testserver/patches/server-remapped/0740-Fix-and-optimise-world-force-upgrading.patch

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Spottedleaf <Spottedleaf@users.noreply.github.com>
Date: Thu, 20 May 2021 07:02:22 -0700
Subject: [PATCH] Fix and optimise world force upgrading

The WorldUpgrader class was incorrectly modified by
CB. It will store an IChunkLoader instance for all
dimension types in the world, but obviously with how
CB shifts around worlds only one dimension type exists
per world. But this would be OK if CB did this
change correctly. All IChunkLoader instances
will point to the same regionfiles. And all
IChunkLoader instances are going to be read from.

This problem hasn't really been reported because
it relies on the persistent legacy data to be converted
as well to cause corruption. Why? Because the legacy
data is also shared, it will result in different
outputs from conversion (as once conversion for legacy
persistent data takes place, it is REMOVED - so the next
convert will _not_ have the data). Which means different
sizes on disk. Which means different regionfile sector
allocations. Which means there are 3 different possible
regionfile sector allocations in memory, and none of them
are going to be correct.

I've fixed this by writing a world upgrader suited to
CB's changes to world folder format. It was brain dead
easy to add threading, so I did.

diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
new file mode 100644
index 0000000000000000000000000000000000000000..f476f21bcfd64d4eb2b690c9100275093c49c9d6
--- /dev/null
+++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java
@@ -0,0 +1,200 @@
+package io.papermc.paper.world;
+
+import com.mojang.datafixers.DataFixer;
+import net.minecraft.SharedConstants;
+import net.minecraft.nbt.CompoundTag;
+import net.minecraft.resources.ResourceKey;
+import net.minecraft.util.worldupdate.WorldUpgrader;
+import net.minecraft.world.level.ChunkPos;
+import net.minecraft.world.level.chunk.storage.ChunkStorage;
+import net.minecraft.world.level.chunk.storage.RegionFileStorage;
+import net.minecraft.world.level.dimension.DimensionType;
+import net.minecraft.world.level.dimension.LevelStem;
+import net.minecraft.world.level.storage.DimensionDataStorage;
+import net.minecraft.world.level.storage.LevelStorageSource;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import java.io.File;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+
+public class ThreadedWorldUpgrader {
+
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    private final ResourceKey<LevelStem> dimensionType;
+    private final ResourceKey<DimensionType> worldKey;
+    private final String worldName;
+    private final ExecutorService threadPool;
+    private final DataFixer dataFixer;
+    private final boolean removeCaches;
+
+    public ThreadedWorldUpgrader(final ResourceKey<LevelStem> dimensionType, final ResourceKey<DimensionType> worldKey, final String worldName, final int threads,
+                                 final DataFixer dataFixer, final boolean removeCaches) {
+        this.dimensionType = dimensionType;
+        this.worldKey = worldKey;
+        this.worldName = worldName;
+        this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() {
+            private final AtomicInteger threadCounter = new AtomicInteger();
+
+            @Override
+            public Thread newThread(final Runnable run) {
+                final Thread ret = new Thread(run);
+
+                ret.setName("World upgrader thread for world " + ThreadedWorldUpgrader.this.worldName + " #" + this.threadCounter.getAndIncrement());
+                ret.setUncaughtExceptionHandler((thread, throwable) -> {
+                    LOGGER.fatal("Error upgrading world", throwable);
+                });
+
+                return ret;
+            }
+        });
+        this.dataFixer = dataFixer;
+        this.removeCaches = removeCaches;
+    }
+
+    public void convert() {
+        final File worldFolder = LevelStorageSource.getFolder(new File(this.worldName), this.dimensionType);
+        final DimensionDataStorage worldPersistentData = new DimensionDataStorage(new File(worldFolder, "data"), this.dataFixer);
+
+        final File regionFolder = new File(worldFolder, "region");
+
+        LOGGER.info("Force upgrading " + this.worldName);
+        LOGGER.info("Counting regionfiles for " + this.worldName);
+        final File[] regionFiles = regionFolder.listFiles((final File dir, final String name) -> {
+            return WorldUpgrader.getRegionfileRegex().matcher(name).matches();
+        });
+        if (regionFiles == null) {
+            LOGGER.info("Found no regionfiles to convert for world " + this.worldName);
+            return;
+        }
+        LOGGER.info("Found " + regionFiles.length + " regionfiles to convert");
+        LOGGER.info("Starting conversion now for world " + this.worldName);
+
+        final WorldInfo info = new WorldInfo(() -> worldPersistentData,
+                new ChunkStorage(regionFolder, this.dataFixer, false), this.removeCaches, this.worldKey);
+
+        long expectedChunks = (long)regionFiles.length * (32L * 32L);
+
+        for (final File regionFile : regionFiles) {
+            final ChunkPos regionPos = RegionFileStorage.getRegionFileCoordinates(regionFile);
+            if (regionPos == null) {
+                expectedChunks -= (32L * 32L);
+                continue;
+            }
+
+            this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5));
+        }
+        this.threadPool.shutdown();
+
+        final DecimalFormat format = new DecimalFormat("#0.00");
+
+        final long start = System.nanoTime();
+
+        while (!this.threadPool.isTerminated()) {
+            final long current = info.convertedChunks.get();
+
+            LOGGER.info("{}% completed ({} / {} chunks)...", format.format((double)current / (double)expectedChunks * 100.0), current, expectedChunks);
+
+            try {
+                Thread.sleep(1000L);
+            } catch (final InterruptedException ignore) {}
+        }
+
+        final long end = System.nanoTime();
+
+        try {
+            info.loader.close();
+        } catch (final IOException ex) {
+            LOGGER.fatal("Failed to close chunk loader", ex);
+        }
+        LOGGER.info("Completed conversion. Took {}s, {} out of {} chunks needed to be converted/modified ({}%)",
+                (int)Math.ceil((end - start) * 1.0e-9), info.modifiedChunks.get(), expectedChunks, format.format((double)info.modifiedChunks.get() / (double)expectedChunks * 100.0));
+    }
+
+    private static final class WorldInfo {
+
+        public final Supplier<DimensionDataStorage> persistentDataSupplier;
+        public final ChunkStorage loader;
+        public final boolean removeCaches;
+        public final ResourceKey<DimensionType> worldKey;
+        public final AtomicLong convertedChunks = new AtomicLong();
+        public final AtomicLong modifiedChunks = new AtomicLong();
+
+        private WorldInfo(final Supplier<DimensionDataStorage> persistentDataSupplier, final ChunkStorage loader, final boolean removeCaches,
+                          final ResourceKey<DimensionType> worldKey) {
+            this.persistentDataSupplier = persistentDataSupplier;
+            this.loader = loader;
+            this.removeCaches = removeCaches;
+            this.worldKey = worldKey;
+        }
+    }
+
+    private static final class ConvertTask implements Runnable {
+
+        private final WorldInfo worldInfo;
+        private final int regionX;
+        private final int regionZ;
+
+        public ConvertTask(final WorldInfo worldInfo, final int regionX, final int regionZ) {
+            this.worldInfo = worldInfo;
+            this.regionX = regionX;
+            this.regionZ = regionZ;
+        }
+
+        @Override
+        public void run() {
+            final int regionCX = this.regionX << 5;
+            final int regionCZ = this.regionZ << 5;
+
+            final Supplier<DimensionDataStorage> persistentDataSupplier = this.worldInfo.persistentDataSupplier;
+            final ChunkStorage loader = this.worldInfo.loader;
+            final boolean removeCaches = this.worldInfo.removeCaches;
+            final ResourceKey<DimensionType> worldKey = this.worldInfo.worldKey;
+
+            for (int cz = regionCZ; cz < (regionCZ + 32); ++cz) {
+                for (int cx = regionCX; cx < (regionCX + 32); ++cx) {
+                    final ChunkPos chunkPos = new ChunkPos(cx, cz);
+                    try {
+                        // no need to check the coordinate of the chunk, the regionfilecache does that for us
+
+                        CompoundTag chunkNBT = loader.read(chunkPos);
+
+                        if (chunkNBT == null) {
+                            continue;
+                        }
+
+                        final int versionBefore = ChunkStorage.getVersion(chunkNBT);
+
+                        chunkNBT = loader.getChunkData(worldKey, persistentDataSupplier, chunkNBT, chunkPos, null);
+
+                        boolean modified = versionBefore < SharedConstants.getCurrentVersion().getWorldVersion();
+
+                        if (removeCaches) {
+                            final CompoundTag level = chunkNBT.getCompound("Level");
+                            modified |= level.contains("Heightmaps");
+                            level.remove("Heightmaps");
+                            modified |= level.contains("isLightOn");
+                            level.remove("isLightOn");
+                        }
+
+                        if (modified) {
+                            this.worldInfo.modifiedChunks.getAndIncrement();
+                            loader.write(chunkPos, chunkNBT);
+                        }
+                    } catch (final Exception ex) {
+                        LOGGER.error("Error upgrading chunk {}", chunkPos, ex);
+                    } finally {
+                        this.worldInfo.convertedChunks.getAndIncrement();
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java
index ba6c17da4875c3a342da99e354c9f07cc7f17326..d0105061c714af4403b3e14e96e54e17a82c172b 100644
--- a/src/main/java/net/minecraft/server/Main.java
+++ b/src/main/java/net/minecraft/server/Main.java
@@ -15,6 +15,7 @@ import java.nio.file.Paths;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.BooleanSupplier;
+import io.papermc.paper.world.ThreadedWorldUpgrader;
 import joptsimple.NonOptionArgumentSpec;
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
@@ -269,6 +270,15 @@ public class Main {
     }
     // Paper end
 
+    // Paper start - fix and optimise world upgrading
+    public static void convertWorldButItWorks(ResourceKey<LevelStem> dimensionType, ResourceKey<DimensionType> worldKey, String worldName,
+                                              DataFixer dataFixer, boolean removeCaches) {
+        int threads = Runtime.getRuntime().availableProcessors() * 3 / 8;
+        final ThreadedWorldUpgrader worldUpgrader = new ThreadedWorldUpgrader(dimensionType, worldKey, worldName, threads, dataFixer, removeCaches);
+        worldUpgrader.convert();
+    }
+    // Paper end - fix and optimise world upgrading
+
     public static void forceUpgrade(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, boolean eraseCache, BooleanSupplier booleansupplier, ImmutableSet<ResourceKey<DimensionType>> worlds) { // CraftBukkit
         Main.LOGGER.info("Forcing world upgrade! {}", session.getLevelId()); // CraftBukkit
         WorldUpgrader worldupgrader = new WorldUpgrader(session, dataFixer, worlds, eraseCache);
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index fd76d776c7003585c9efef44c6d7da0f6c3f574e..9d7cebd703bd0171ca3e95d2985c1a52fdb59712 100644
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
@@ -512,13 +512,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
                 worlddata = new PrimaryLevelData(worldsettings, generatorsettings, Lifecycle.stable());
             }
             worlddata.checkName(name); // CraftBukkit - Migration did not rewrite the level.dat; This forces 1.8 to take the last loaded world as respawn (in this case the end)
-            if (options.has("forceUpgrade")) {
-                net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), options.has("eraseCache"), () -> {
-                    return true;
-                }, worlddata.worldGenSettings().dimensions().entrySet().stream().map((entry1) -> {
-                    return ResourceKey.create(Registry.DIMENSION_TYPE_REGISTRY, ((ResourceKey) entry1.getKey()).location());
-                }).collect(ImmutableSet.toImmutableSet()));
-            }
+            // Paper - move down
 
             ServerLevelData iworlddataserver = worlddata;
             WorldGenSettings generatorsettings = worlddata.worldGenSettings();
@@ -538,6 +532,14 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
                 chunkgenerator = worlddimension.generator();
             }
 
+            // Paper start - fix and optimise world upgrading
+            if (options.has("forceUpgrade")) {
+                net.minecraft.server.Main.convertWorldButItWorks(
+                        dimensionKey, Level.getDimensionKey(dimensionmanager), worldSession.getLevelId(), DataFixers.getDataFixer(), options.has("eraseCache")
+                );
+            }
+            // Paper end - fix and optimise world upgrading
+
             ResourceKey<Level> worldKey = ResourceKey.create(Registry.DIMENSION_REGISTRY, dimensionKey.location());
 
             if (dimensionKey == LevelStem.OVERWORLD) {
diff --git a/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java b/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java
index 79491c5081dbc0cc479d6bc0329ff9b374559c9b..29c0fe698e91e232bcebfafdd853d222c83d5af8 100644
--- a/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java
+++ b/src/main/java/net/minecraft/util/worldupdate/WorldUpgrader.java
@@ -52,7 +52,7 @@ public class WorldUpgrader {
     private volatile int skipped;
     private final Object2FloatMap<ResourceKey<DimensionType>> progressMap = Object2FloatMaps.synchronize(new Object2FloatOpenCustomHashMap(Util.identityStrategy())); // CraftBukkit
     private volatile Component status = new TranslatableComponent("optimizeWorld.stage.counting");
-    private static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$");
+    private static final Pattern REGEX = Pattern.compile("^r\\.(-?[0-9]+)\\.(-?[0-9]+)\\.mca$"); public static final Pattern getRegionfileRegex() { return REGEX; } // Paper - OBFHELPER
     private final DimensionDataStorage overworldDataStorage;
 
     public WorldUpgrader(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, ImmutableSet<ResourceKey<DimensionType>> worlds, boolean eraseCache) { // CraftBukkit
diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
index 4523bc1f49e7be248a47eeb599fa7b6550dbb08d..d155d00abf8f423e64e6e6d80ddadbc1cfb58a64 100644
--- a/src/main/java/net/minecraft/world/level/Level.java
+++ b/src/main/java/net/minecraft/world/level/Level.java
@@ -181,6 +181,15 @@ public abstract class Level implements LevelAccessor, AutoCloseable {
         return typeKey;
     }
 
+    // Paper start - fix and optimise world upgrading
+    // copied from below
+    public static ResourceKey<DimensionType> getDimensionKey(DimensionType manager) {
+        return ((org.bukkit.craftbukkit.CraftServer)org.bukkit.Bukkit.getServer()).getHandle().getServer().registryHolder.dimensionTypes().getResourceKey(manager).orElseThrow(() -> {
+            return new IllegalStateException("Unregistered dimension type: " + manager);
+        });
+    }
+    // Paper end - fix and optimise world upgrading
+
     protected Level(WritableLevelData worlddatamutable, ResourceKey<Level> resourcekey, final DimensionType dimensionmanager, Supplier<ProfilerFiller> supplier, boolean flag, boolean flag1, long i, org.bukkit.generator.ChunkGenerator gen, org.bukkit.World.Environment env, java.util.concurrent.Executor executor) { // Paper
         this.spigotConfig = new org.spigotmc.SpigotWorldConfig(((net.minecraft.world.level.storage.PrimaryLevelData) worlddatamutable).getLevelName()); // Spigot
         this.paperConfig = new com.destroystokyo.paper.PaperWorldConfig(((net.minecraft.world.level.storage.PrimaryLevelData) worlddatamutable).getLevelName(), this.spigotConfig); // Paper
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java
index 1af804c5c6fb2b20ea3f020610763c1d7dcee110..0e38f2f31d167c417b707f00aa68cacaef3d9f6c 100644
--- a/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/ChunkStorage.java
@@ -109,6 +109,7 @@ public class ChunkStorage implements AutoCloseable {
         return nbttagcompound;
     }
 
+    public static int getVersion(CompoundTag nbttagcompound) { return getVersion(nbttagcompound); } // Paper - OBFHELPER
     public static int getVersion(CompoundTag tag) {
         return tag.contains("DataVersion", 99) ? tag.getInt("DataVersion") : -1;
     }
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
index 0498982ac14f20145d68dbf64a46bcaacf5516ef..7a01f2fbe459e36cee5416455a049b25963e257a 100644
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
@@ -30,6 +30,28 @@ public class RegionFileStorage implements AutoCloseable { // Paper - no final
 
 
     // Paper start
+    public static ChunkPos getRegionFileCoordinates(File file) {
+        String fileName = file.getName();
+        if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) {
+            return null;
+        }
+
+        String[] split = fileName.split("\\.");
+
+        if (split.length != 4) {
+            return null;
+        }
+
+        try {
+            int x = Integer.parseInt(split[1]);
+            int z = Integer.parseInt(split[2]);
+
+            return new ChunkPos(x << 5, z << 5);
+        } catch (NumberFormatException ex) {
+            return null;
+        }
+    }
+
     public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) { // Paper - synchronize for async io
         return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()));
     }
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
index f1e6d0050092ad51bf233c80b6a51a121e961b07..831f187cde88e815c9a859e52ab45fbbe054f83e 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
@@ -1145,14 +1145,7 @@ public final class CraftServer implements Server {
         }
         worlddata.checkName(name);
         worlddata.setModdedInfo(console.getServerModName(), console.getModdedStatus().isPresent());
-
-        if (console.options.has("forceUpgrade")) {
-            net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), console.options.has("eraseCache"), () -> {
-                return true;
-            }, worlddata.worldGenSettings().dimensions().entrySet().stream().map((entry) -> {
-                return ResourceKey.create(Registry.DIMENSION_TYPE_REGISTRY, ((ResourceKey) entry.getKey()).location());
-            }).collect(ImmutableSet.toImmutableSet()));
-        }
+        // Paper - move down
 
         long j = BiomeManager.obfuscateSeed(creator.seed());
         List<CustomSpawner> list = ImmutableList.of(new PhantomSpawner(), new PatrolSpawner(), new CatSpawner(), new VillageSiege(), new WanderingTraderSpawner(worlddata));
@@ -1169,6 +1162,14 @@ public final class CraftServer implements Server {
             chunkgenerator = worlddimension.generator();
         }
 
+        // Paper start - fix and optimise world upgrading
+        if (console.options.has("forceUpgrade")) {
+            net.minecraft.server.Main.convertWorldButItWorks(
+                actualDimension, net.minecraft.world.level.Level.getDimensionKey(dimensionmanager), worldSession.getLevelId(), DataFixers.getDataFixer(), console.options.has("eraseCache")
+            );
+        }
+        // Paper end - fix and optimise world upgrading
+
         ResourceKey<net.minecraft.world.level.Level> worldKey;
         String levelName = this.getServer().getProperties().levelName;
         if (name.equals(levelName + "_nether")) {