testserver/Spigot-Server-Patches/0416-Allow-Saving-of-Oversized-Chunks.patch

From bcbc92e8794aa43fefd2c9b0eb52cd2cee2b2cb2 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Fri, 15 Feb 2019 01:08:19 -0500
Subject: [PATCH] Allow Saving of Oversized Chunks

The Minecraft World Region File format has a hard cap of 1MB per chunk.
This is due to the fact that the header of the file format only allocates
a single byte for sector count, meaning a maximum of 256 sectors, at 4k per sector.

This limit can be reached fairly easily with books, resulting in the chunk being unable
to save to the world. Worse off, is that nothing printed when this occured, and silently
performed a chunk rollback on next load.

This leads to security risk with duplication and is being actively exploited.

This patch catches the too large scenario, falls back and moves any large Entity
or Tile Entity into a new compound, and this compound is saved into a different file.

On Chunk Load, we check for oversized status, and if so, we load the extra file and
merge the Entities and Tile Entities from the oversized chunk back into the level to
then be loaded as normal.

Once a chunk is returned back to normal size, the oversized flag will clear, and no
extra data file will exist.

This fix maintains compatability with all existing Anvil Region Format tools as it
does not alter the save format. They will just not know about the extra entities.

This fix also maintains compatability if someone switches server jars to one without
this fix, as the data will remain in the oversized file. Once the server returns
to a jar with this fix, the data will be restored.

diff --git a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
index 12268f87b..e1f7e06ab 100644
--- a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
+++ b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
@@ -39,6 +39,7 @@ public class NBTCompressedStreamTools {
 
     }
 
+    public static NBTTagCompound readNBT(DataInputStream datainputstream) throws IOException { return a(datainputstream); } // Paper - OBFHELPER
     public static NBTTagCompound a(DataInputStream datainputstream) throws IOException {
         return a((DataInput) datainputstream, NBTReadLimiter.a);
     }
@@ -59,6 +60,7 @@ public class NBTCompressedStreamTools {
         }
     }
 
+    public static void writeNBT(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException { a(nbttagcompound, dataoutput); } // Paper - OBFHELPER
     public static void a(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException {
         a((NBTBase) nbttagcompound, dataoutput);
     }
diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
index c20511588..d148ce497 100644
--- a/src/main/java/net/minecraft/server/RegionFile.java
+++ b/src/main/java/net/minecraft/server/RegionFile.java
@@ -78,6 +78,7 @@ public class RegionFile {
             }
             header.clear();
             java.nio.IntBuffer headerAsInts = header.asIntBuffer();
+            initOversizedState();
             // Paper End
 
             for (j = 0; j < 1024; ++j) {
@@ -93,7 +94,7 @@ public class RegionFile {
                         this.c.seek(j * 4 + 4); // Go back to where we were
                     }
                 }
-                if (k > 0 && (k >> 8) > 1 && (k >> 8) + (k & 255) <= this.f.size()) { // Paper >= 1 as 0/1 are the headers, and negative isnt valid
+                if (k > 0 && (k >> 8) > 1 && (k >> 8) + (length) <= this.f.size()) { // Paper >= 1 as 0/1 are the headers, and negative isnt valid
                     for (int l = 0; l < (length); ++l) {
                         // Spigot end
                         this.f.set((k >> 8) + l, false);
@@ -119,7 +120,7 @@ public class RegionFile {
     }
 
     @Nullable
-    public synchronized DataInputStream a(int i, int j) {
+    public synchronized DataInputStream getReadStream(int i, int j) { return a(i, j); } @Nullable public synchronized DataInputStream a(int i, int j) { // Paper - OBFHELPER
         if (this.e(i, j)) {
             return null;
         } else {
@@ -203,8 +204,8 @@ public class RegionFile {
     }
 
     @Nullable
-    public DataOutputStream c(int i, int j) {
-        return this.e(i, j) ? null : new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(new RegionFile.ChunkBuffer(i, j))));
+    public DataOutputStream getWriteStream(int i, int j) { return c(i, j); } @Nullable public DataOutputStream c(int i, int j) { // Paper - OBFHELPER
+        return this.e(i, j) ? null : new DataOutputStream(new RegionFile.ChunkBuffer(i, j)); // Paper - remove middleware, move deflate to .close() for dynamic levels
     }
 
     protected synchronized void a(int i, int j, byte[] abyte, int k) {
@@ -222,8 +223,9 @@ public class RegionFile {
 
             if (k1 >= 256) {
                 // Spigot start
-                if (!ENABLE_EXTENDED_SAVE) return;
+                if (!USE_SPIGOT_OVERSIZED_METHOD) throw new ChunkTooLargeException(i, j, k1); // Paper - throw error instead
                 org.bukkit.Bukkit.getLogger().log(java.util.logging.Level.WARNING,"Large Chunk Detected: ({0}, {1}) Size: {2} {3}", new Object[]{i, j, k1, this.b});
+                if (!ENABLE_EXTENDED_SAVE) return;
                 // Spigot end
             }
 
@@ -374,6 +376,109 @@ public class RegionFile {
             logger.error("Error backing up corrupt file" + file.getAbsolutePath(), e);
         }
     }
+
+    private final byte[] oversized = new byte[1024];
+    private int oversizedCount = 0;
+
+    private synchronized void initOversizedState() throws IOException {
+        File metaFile = getOversizedMetaFile();
+        if (metaFile.exists()) {
+            final byte[] read = java.nio.file.Files.readAllBytes(metaFile.toPath());
+            System.arraycopy(read, 0, oversized, 0, oversized.length);
+            for (byte temp : oversized) {
+                oversizedCount += temp;
+            }
+        }
+    }
+
+    private static int getChunkIndex(int x, int z) {
+        return (x & 31) + (z & 31) * 32;
+    }
+    synchronized boolean isOversized(int x, int z) {
+        return this.oversized[getChunkIndex(x, z)] == 1;
+    }
+    synchronized void setOversized(int x, int z, boolean oversized) throws IOException {
+        final int offset = getChunkIndex(x, z);
+        boolean previous = this.oversized[offset] == 1;
+        this.oversized[offset] = (byte) (oversized ? 1 : 0);
+        if (!previous && oversized) {
+            oversizedCount++;
+        } else if (!oversized && previous) {
+            oversizedCount--;
+        }
+        if (previous && !oversized) {
+            File oversizedFile = getOversizedFile(x, z);
+            if (oversizedFile.exists()) {
+                oversizedFile.delete();
+            }
+        }
+        if (oversizedCount > 0) {
+            if (previous != oversized) {
+                writeOversizedMeta();
+            }
+        } else if (previous) {
+            File oversizedMetaFile = getOversizedMetaFile();
+            if (oversizedMetaFile.exists()) {
+                oversizedMetaFile.delete();
+            }
+        }
+    }
+
+    private void writeOversizedMeta() throws IOException {
+        java.nio.file.Files.write(getOversizedMetaFile().toPath(), oversized);
+    }
+
+    private File getOversizedMetaFile() {
+        return new File(getFile().getParentFile(), getFile().getName().replaceAll("\\.mca$", "") + ".oversized.nbt");
+    }
+
+    private File getOversizedFile(int x, int z) {
+        return new File(this.getFile().getParentFile(), this.getFile().getName().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt");
+    }
+
+    void writeOversizedData(int x, int z, NBTTagCompound oversizedData) throws IOException {
+        File file = getOversizedFile(x, z);
+        try (DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new DeflaterOutputStream(new java.io.FileOutputStream(file), new java.util.zip.Deflater(java.util.zip.Deflater.BEST_COMPRESSION), 32 * 1024), 32 * 1024))) {
+            NBTCompressedStreamTools.writeNBT(oversizedData, out);
+        }
+        this.setOversized(x, z, true);
+
+    }
+
+    synchronized NBTTagCompound getOversizedData(int x, int z) throws IOException {
+        File file = getOversizedFile(x, z);
+        try (DataInputStream out = new DataInputStream(new BufferedInputStream(new InflaterInputStream(new java.io.FileInputStream(file))))) {
+            return NBTCompressedStreamTools.readNBT(out);
+        }
+
+    }
+
+    private static final boolean USE_SPIGOT_OVERSIZED_METHOD = Boolean.getBoolean("Paper.useSpigotExtendedSaveMethod"); // Paper
+    static {
+        if (USE_SPIGOT_OVERSIZED_METHOD) {
+            org.bukkit.Bukkit.getLogger().log(java.util.logging.Level.SEVERE, "====================================");
+            org.bukkit.Bukkit.getLogger().log(java.util.logging.Level.SEVERE, "Using Spigot Oversized Chunk save method. Warning this will result in extremely fragmented chunks, as well as making the entire region file unable to be to used in any other software but Forge or Spigot (not usable in Vanilla or CraftBukkit). Paper's method is highly recommended.");
+            org.bukkit.Bukkit.getLogger().log(java.util.logging.Level.SEVERE, "====================================");
+        }
+    }
+    public class ChunkTooLargeException extends RuntimeException {
+        public ChunkTooLargeException(int x, int z, int sectors) {
+            super("Chunk " + x + "," + z + " of " + getFile().toString() + " is too large (" + sectors + "/256)");
+        }
+    }
+    private static class DirectByteArrayOutputStream extends ByteArrayOutputStream {
+        public DirectByteArrayOutputStream() {
+            super();
+        }
+
+        public DirectByteArrayOutputStream(int size) {
+            super(size);
+        }
+
+        public byte[] getBuffer() {
+            return this.buf;
+        }
+    }
     // Paper end
 
     class ChunkBuffer extends ByteArrayOutputStream {
@@ -387,8 +492,40 @@ public class RegionFile {
             this.c = j;
         }
 
-        public void close() {
-            RegionFile.this.a(this.b, this.c, this.buf, this.count);
+        public void close() throws IOException {
+            // Paper start - apply dynamic compression
+            int origLength = this.count;
+            byte[] buf = this.buf;
+            DirectByteArrayOutputStream out = compressData(buf, origLength);
+            byte[] bytes = out.getBuffer();
+            int length = out.size();
+
+            RegionFile.this.a(this.b, this.c, bytes, length); // Paper - change to bytes/length
+            // Paper end
+        }
+    }
+
+    private static DirectByteArrayOutputStream compressData(byte[] buf, int length) throws IOException {
+        final java.util.zip.Deflater deflater;
+        if (length > 1024 * 512) {
+            deflater = new java.util.zip.Deflater(9);
+        } else if (length > 1024 * 128) {
+            deflater = new java.util.zip.Deflater(8);
+        } else {
+            deflater = new java.util.zip.Deflater(6);
+        }
+
+
+        deflater.setInput(buf, 0, length);
+        deflater.finish();
+
+        DirectByteArrayOutputStream out = new DirectByteArrayOutputStream(length);
+        byte[] buffer = new byte[1024 * (length > 1024 * 124 ? 32 : 16)];
+        while (!deflater.finished()) {
+            out.write(buffer, 0, deflater.deflate(buffer));
         }
+        out.close();
+        deflater.end();
+        return out;
     }
 }
diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
index 8c8b7cbab..a17e76d83 100644
--- a/src/main/java/net/minecraft/server/RegionFileCache.java
+++ b/src/main/java/net/minecraft/server/RegionFileCache.java
@@ -16,6 +16,7 @@ public class RegionFileCache {
 
     public static final Map<File, RegionFile> cache = new LinkedHashMap(PaperConfig.regionFileCacheSize, 0.75f, true); // Paper - HashMap -> LinkedHashMap
 
+    public static synchronized RegionFile getRegionFile(File file, int i, int j) { return a(file, i, j); } // Paper - OBFHELPER
     public static synchronized RegionFile a(File file, int i, int j) {
         File file1 = new File(file, "region");
         File file2 = new File(file1, "r." + (i >> 5) + "." + (j >> 5) + ".mca");
@@ -83,6 +84,125 @@ public class RegionFileCache {
     public static synchronized boolean hasRegionFile(File file, int i, int j) {
         return RegionFileCache.cache.containsKey(getRegionFileName(file, i, j));
     }
+    private static void printOversizedLog(String msg, File file, int x, int z) {
+        org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed.");
+    }
+
+    private static final int DEFAULT_SIZE_THRESHOLD = 1024 * 8;
+    private static final int OVERZEALOUS_THRESHOLD = 1024 * 2;
+    private static int SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD;
+    private static void resetFilterThresholds() {
+        SIZE_THRESHOLD = Math.max(1024 * 4, Integer.getInteger("Paper.FilterThreshhold", DEFAULT_SIZE_THRESHOLD));
+    }
+    static {
+        resetFilterThresholds();
+    }
+    private static void writeRegion(File file, int x, int z, NBTTagCompound nbttagcompound) throws IOException {
+        RegionFile regionfile = getRegionFile(file, x, z);
+
+        DataOutputStream out = regionfile.getWriteStream(x & 31, z & 31);
+        try {
+            NBTCompressedStreamTools.writeNBT(nbttagcompound, out);
+            out.close();
+            regionfile.setOversized(x, z, false);
+        } catch (RegionFile.ChunkTooLargeException ignored) {
+            printOversizedLog("ChunkTooLarge! Someone is trying to duplicate.", file, x, z);
+            // Clone as we are now modifying it, don't want to corrupt the pending save state
+            nbttagcompound = nbttagcompound.clone();
+            // Filter out TileEntities and Entities
+            NBTTagCompound oversizedData = filterChunkData(nbttagcompound);
+            //noinspection SynchronizationOnLocalVariableOrMethodParameter
+            synchronized (regionfile) {
+                out = regionfile.getWriteStream(x & 31, z & 31);
+                NBTCompressedStreamTools.writeNBT(nbttagcompound, out);
+                try {
+                    out.close();
+                    // 2048 is below the min allowed, so it means we enter overzealous mode below
+                    if (SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD) {
+                        resetFilterThresholds();
+                    }
+                } catch (RegionFile.ChunkTooLargeException e) {
+                    printOversizedLog("ChunkTooLarge even after reduction. Trying in overzealous mode.", file, x, z);
+                    // Eek, major fail. We have retry logic, so reduce threshholds and fall back
+                    SIZE_THRESHOLD = OVERZEALOUS_THRESHOLD;
+                    throw e;
+                }
+
+                regionfile.writeOversizedData(x, z, oversizedData);
+            }
+        }
+    }
+
+    private static NBTTagCompound filterChunkData(NBTTagCompound chunk) {
+        NBTTagCompound oversizedLevel = new NBTTagCompound();
+        NBTTagCompound level = chunk.getCompound("Level");
+        filterChunkList(level, oversizedLevel, "Entities");
+        filterChunkList(level, oversizedLevel, "TileEntities");
+        NBTTagCompound oversized = new NBTTagCompound();
+        oversized.set("Level", oversizedLevel);
+        return oversized;
+    }
+
+    private static void filterChunkList(NBTTagCompound level, NBTTagCompound extra, String key) {
+        NBTTagList list = level.getList(key, 10);
+        NBTTagList newList = extra.getList(key, 10);
+        for (Iterator<NBTBase> iterator = list.list.iterator(); iterator.hasNext(); ) {
+            NBTBase object = iterator.next();
+            if (getNBTSize(object) > SIZE_THRESHOLD) {
+                newList.add(object);
+                iterator.remove();
+            }
+        }
+        level.set(key, list);
+        extra.set(key, newList);
+    }
+
+
+    private static NBTTagCompound readOversizedChunk(RegionFile regionfile, int i, int j) throws IOException {
+        synchronized (regionfile) {
+            try (DataInputStream datainputstream = regionfile.getReadStream(i & 31, j & 31)) {
+                NBTTagCompound oversizedData = regionfile.getOversizedData(i, j);
+                NBTTagCompound chunk = NBTCompressedStreamTools.readNBT(datainputstream);
+                if (oversizedData == null) {
+                    return chunk;
+                }
+                NBTTagCompound oversizedLevel = oversizedData.getCompound("Level");
+                NBTTagCompound level = chunk.getCompound("Level");
+
+                mergeChunkList(level, oversizedLevel, "Entities");
+                mergeChunkList(level, oversizedLevel, "TileEntities");
+
+                chunk.set("Level", level);
+
+                return chunk;
+            } catch (Throwable throwable) {
+                throwable.printStackTrace();
+                throw throwable;
+            }
+        }
+    }
+
+    private static void mergeChunkList(NBTTagCompound level, NBTTagCompound oversizedLevel, String key) {
+        NBTTagList levelList = level.getList(key, 10);
+        NBTTagList oversizedList = oversizedLevel.getList(key, 10);
+
+        if (!oversizedList.isEmpty()) {
+            levelList.addAll(oversizedList);
+            level.set(key, levelList);
+        }
+    }
+
+    private static int getNBTSize(NBTBase nbtBase) {
+        DataOutputStream test = new DataOutputStream(new org.apache.commons.io.output.NullOutputStream());
+        try {
+            nbtBase.write(test);
+            return test.size();
+        } catch (IOException e) {
+            e.printStackTrace();
+            return 0;
+        }
+    }
+
     // Paper End
 
     public static synchronized void a() {
@@ -108,6 +228,12 @@ public class RegionFileCache {
     // CraftBukkit start - call sites hoisted for synchronization
     public static NBTTagCompound read(File file, int i, int j) throws IOException { // Paper - remove synchronization
         RegionFile regionfile = a(file, i, j);
+        // Paper start
+        if (regionfile.isOversized(i, j)) {
+            printOversizedLog("Loading Oversized Chunk!", file, i, j);
+            return readOversizedChunk(regionfile, i, j);
+        }
+        // Paper end
 
         DataInputStream datainputstream = regionfile.a(i & 31, j & 31);
 
@@ -121,11 +247,14 @@ public class RegionFileCache {
     @Nullable
     public static void write(File file, int i, int j, NBTTagCompound nbttagcompound) throws IOException {
         int attempts = 0; Exception laste = null; while (attempts++ < 5) { try { // Paper
-        RegionFile regionfile = a(file, i, j);
-
-        DataOutputStream dataoutputstream = regionfile.c(i & 31, j & 31);
-        NBTCompressedStreamTools.a(nbttagcompound, (java.io.DataOutput) dataoutputstream);
-        dataoutputstream.close();
+        writeRegion(file, i, j, nbttagcompound); // Paper - moved to own method
+        // Paper start
+//      RegionFile regionfile = a(file, i, j);
+//
+//      DataOutputStream dataoutputstream = regionfile.c(i & 31, j & 31);
+//      NBTCompressedStreamTools.a(nbttagcompound, (java.io.DataOutput) dataoutputstream);
+//      dataoutputstream.close();
+        // Paper end
         // Paper start
             laste = null; break; // Paper
             } catch (Exception exception) {
-- 
2.20.1