testserver/Spigot-Server-Patches/0446-Improved-Watchdog-Support.patch

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Sun, 12 Apr 2020 15:50:48 -0400
Subject: [PATCH] Improved Watchdog Support

Forced Watchdog Crash support and Improve Async Shutdown

If the request to shut down the server is received while we are in
a watchdog hang, immediately treat it as a crash and begin the shutdown
process. Shutdown process is now improved to also shutdown cleanly when
not using restart scripts either.

If a server is deadlocked, a server owner can send SIGUP (or any other signal
the JVM understands to shut down as it currently does) and the watchdog
will no longer need to wait until the full timeout, allowing you to trigger
a close process and try to shut the server down gracefully, saving player and
world data.

Previously there was no way to trigger this outside of waiting for a full watchdog
timeout, which may be set to a really long time...

Additionally, fix everything to do with shutting the server down asynchronously.

Previously, nearly everything about the process was fragile and unsafe. Main might
not have actually been frozen, and might still be manipulating state.

Or, some reuest might ask main to do something in the shutdown but main is dead.

Or worse, other things might start closing down items such as the Console or Thread Pool
before we are fully shutdown.

This change tries to resolve all of these issues by moving everything into the stop
method and guaranteeing only one thread is stopping the server.

We then issue Thread Death to the main thread of another thread initiates the stop process.
We have to ensure Thread Death propagates correctly though to stop main completely.

This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save.

This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they
are properly accounted for and wont trip watchdog on init.

diff --git a/src/main/java/com/destroystokyo/paper/Metrics.java b/src/main/java/com/destroystokyo/paper/Metrics.java
index e257d6b36e0e78dac5b8320017d92776171e1bb0..24641501aeae0f2c2c5ce877775c7b971fb051bf 100644
--- a/src/main/java/com/destroystokyo/paper/Metrics.java
+++ b/src/main/java/com/destroystokyo/paper/Metrics.java
@@ -86,6 +86,9 @@ public class Metrics {
         timer.scheduleAtFixedRate(new TimerTask() {
             @Override
             public void run() {
+                if (MinecraftServer.getServer().hasStopped()) {
+                    return;
+                }
                 submitData();
             }
         }, 1000 * 60 * 5, 1000 * 60 * 30);
diff --git a/src/main/java/net/minecraft/server/CrashReport.java b/src/main/java/net/minecraft/server/CrashReport.java
index 95e6a6de7ccfc4445d0ac19c5f874c0d533b1712..cc6e6f245ee5e73bd570cf42381bf55ee0b364d3 100644
--- a/src/main/java/net/minecraft/server/CrashReport.java
+++ b/src/main/java/net/minecraft/server/CrashReport.java
@@ -257,6 +257,7 @@ public class CrashReport {
     }
 
     public static CrashReport a(Throwable throwable, String s) {
+        if (throwable instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(throwable); // Paper
         while (throwable instanceof CompletionException && throwable.getCause() != null) {
             throwable = throwable.getCause();
         }
diff --git a/src/main/java/net/minecraft/server/DedicatedServer.java b/src/main/java/net/minecraft/server/DedicatedServer.java
index 4fa8d40d84884f17b4bbcc357a6bb902f8f668cd..22ee001a8d5a3c93303d036daa7ee12a6ec802bb 100644
--- a/src/main/java/net/minecraft/server/DedicatedServer.java
+++ b/src/main/java/net/minecraft/server/DedicatedServer.java
@@ -242,7 +242,7 @@ public class DedicatedServer extends MinecraftServer implements IMinecraftServer
             long j = SystemUtils.getMonotonicNanos() - i;
             String s = String.format(Locale.ROOT, "%.3fs", (double) j / 1.0E9D);
 
-            DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s);
+            //DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s); // Paper moved to after init
             if (dedicatedserverproperties.announcePlayerAchievements != null) {
                 ((GameRules.GameRuleBoolean) this.getGameRules().get(GameRules.ANNOUNCE_ADVANCEMENTS)).a(dedicatedserverproperties.announcePlayerAchievements, (MinecraftServer) this);
             }
@@ -368,6 +368,7 @@ public class DedicatedServer extends MinecraftServer implements IMinecraftServer
             //this.remoteStatusListener.b(); // Paper - don't wait for remote connections
         }
 
+        hasFullyShutdown = true; // Paper
         System.exit(0); // CraftBukkit
     }
 
@@ -696,7 +697,7 @@ public class DedicatedServer extends MinecraftServer implements IMinecraftServer
     @Override
     public void stop() {
         super.stop();
-        SystemUtils.h();
+        //SystemUtils.h(); // Paper - moved into super
     }
 
     @Override
diff --git a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java
index 63fdae15ccbef0c39718b320dbd096794bcfa3b4..6beefff203ed6e448898eb5b2e95800def868381 100644
--- a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java
+++ b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java
@@ -135,6 +135,7 @@ public abstract class IAsyncTaskHandler<R extends Runnable> implements Mailbox<R
         try {
             r0.run();
         } catch (Exception exception) {
+            if (exception.getCause() instanceof ThreadDeath) throw exception; // Paper
             IAsyncTaskHandler.LOGGER.fatal("Error executing task on {}", this.bh(), exception);
         }
 
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index 852dc6a5edcf88adcf8d04f1109d7eaff1bea8f9..e4e946c0b3c6b73f02567fffaaabf06312cbea91 100644
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
@@ -160,7 +160,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
     public boolean serverAutoSave = false; // Paper
     public File bukkitDataPackFolder;
     public CommandDispatcher vanillaCommandDispatcher;
-    private boolean forceTicks;
+    public boolean forceTicks; // Paper
     // CraftBukkit end
     // Spigot start
     public static final int TPS = 20;
@@ -170,6 +170,8 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
     public final SlackActivityAccountant slackActivityAccountant = new SlackActivityAccountant();
     // Spigot end
 
+    public volatile Thread shutdownThread; // Paper
+
     public static <S extends MinecraftServer> S a(Function<Thread, S> function) {
         AtomicReference<S> atomicreference = new AtomicReference();
         Thread thread = new Thread(() -> {
@@ -738,6 +740,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
 
     // CraftBukkit start
     private boolean hasStopped = false;
+    public volatile boolean hasFullyShutdown = false; // Paper
     private final Object stopLock = new Object();
     public final boolean hasStopped() {
         synchronized (stopLock) {
@@ -750,10 +753,22 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
         // CraftBukkit start - prevent double stopping on multiple threads
         synchronized(stopLock) {
             if (hasStopped) return;
+            shutdownThread = Thread.currentThread();
             hasStopped = true;
+            org.spigotmc.WatchdogThread.doStop(); // Paper
+            // Paper start - kill main thread, and kill it hard
+            if (!isMainThread()) {
+                while (this.getThread().isAlive()) {
+                    this.getThread().stop();
+                    try {
+                        Thread.sleep(1);
+                    } catch (InterruptedException e) {}
+                }
+            }
+            // Paper end
         }
         // CraftBukkit end
-        MinecraftServer.LOGGER.info("Stopping server");
+        MinecraftServer.LOGGER.info("Stopping server (Ignore any thread death message you see! - DO NOT REPORT THREAD DEATH TO PAPER)"); // Paper
         MinecraftTimings.stopServer(); // Paper
         // CraftBukkit start
         if (this.server != null) {
@@ -817,7 +832,18 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
             this.getUserCache().c(false); // Paper
         }
         // Spigot end
+        // Paper start - move final shutdown items here
+        LOGGER.info("Flushing Chunk IO");
         com.destroystokyo.paper.io.PaperFileIOThread.Holder.INSTANCE.close(true, true); // Paper
+        LOGGER.info("Closing Thread Pool");
+        SystemUtils.shutdownServerThreadPool(); // Paper
+        LOGGER.info("Closing Server");
+        try {
+            net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+        } catch (Exception e) {
+        }
+        this.exit();
+        // Paper end
     }
 
     public String getServerIp() {
@@ -910,6 +936,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
 
     protected void v() {
         try {
+            long serverStartTime = SystemUtils.getMonotonicNanos(); // Paper
             if (this.init()) {
                 this.nextTick = SystemUtils.getMonotonicMillis();
                 this.serverPing.setMOTD(new ChatComponentText(this.motd));
@@ -917,6 +944,18 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
                 this.a(this.serverPing);
 
                 // Spigot start
+                // Paper start - move done tracking
+                LOGGER.info("Running delayed init tasks");
+                this.server.getScheduler().mainThreadHeartbeat(this.ticks); // run all 1 tick delay tasks during init,
+                // this is going to be the first thing the tick process does anyways, so move done and run it after
+                // everything is init before watchdog tick.
+                // anything at 3+ won't be caught here but also will trip watchdog....
+                // tasks are default scheduled at -1 + delay, and first tick will tick at 1
+                String doneTime = String.format(java.util.Locale.ROOT, "%.3fs", (double) (SystemUtils.getMonotonicNanos() - serverStartTime) / 1.0E9D);
+                LOGGER.info("Done ({})! For help, type \"help\"", doneTime);
+                // Paper end
+
+                org.spigotmc.WatchdogThread.tick(); // Paper
                 org.spigotmc.WatchdogThread.hasStarted = true; // Paper
                 Arrays.fill( recentTps, 20 );
                 long start = System.nanoTime(), curTime, tickSection = start; // Paper - Further improve server tick loop
@@ -972,6 +1011,12 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
                 this.a((CrashReport) null);
             }
         } catch (Throwable throwable) {
+            // Paper start
+            if (throwable instanceof ThreadDeath) {
+                MinecraftServer.LOGGER.error("Main thread terminated by WatchDog due to hard crash", throwable);
+                return;
+            }
+            // Paper end
             MinecraftServer.LOGGER.error("Encountered an unexpected exception", throwable);
             // Spigot Start
             if ( throwable.getCause() != null )
@@ -1003,14 +1048,14 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
             } catch (Throwable throwable1) {
                 MinecraftServer.LOGGER.error("Exception stopping the server", throwable1);
             } finally {
-                org.spigotmc.WatchdogThread.doStop(); // Spigot
+                //org.spigotmc.WatchdogThread.doStop(); // Spigot // Paper - move into stop
                 // CraftBukkit start - Restore terminal to original settings
                 try {
-                    net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+                    //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
                 } catch (Exception ignored) {
                 }
                 // CraftBukkit end
-                this.exit();
+                //this.exit(); // Paper - moved into stop
             }
 
         }
@@ -1066,6 +1111,12 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
 
     @Override
     protected TickTask postToMainThread(Runnable runnable) {
+        // Paper start - anything that does try to post to main during watchdog crash, run on watchdog
+        if (this.hasStopped && Thread.currentThread().equals(shutdownThread)) {
+            runnable.run();
+            runnable = () -> {};
+        }
+        // Paper end
         return new TickTask(this.ticks, runnable);
     }
 
@@ -1312,6 +1363,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
                 try {
                     crashreport = CrashReport.a(throwable, "Exception ticking world");
                 } catch (Throwable t) {
+                    if (throwable instanceof ThreadDeath) { throw (ThreadDeath)throwable; } // Paper
                     throw new RuntimeException("Error generating crash report", t);
                 }
                 // Spigot End
diff --git a/src/main/java/net/minecraft/server/PlayerChunkMap.java b/src/main/java/net/minecraft/server/PlayerChunkMap.java
index 9d4c5f32036118bf7c5081c55bea71efd606b8f6..ba100e5c96d5279ab5f30c6e10f3cbd7b6b4443e 100644
--- a/src/main/java/net/minecraft/server/PlayerChunkMap.java
+++ b/src/main/java/net/minecraft/server/PlayerChunkMap.java
@@ -486,6 +486,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
             MutableBoolean mutableboolean = new MutableBoolean();
 
             do {
+                boolean isShuttingDown = world.getMinecraftServer().hasStopped(); // Paper
                 mutableboolean.setFalse();
                 list.stream().map((playerchunk) -> {
                     CompletableFuture completablefuture;
diff --git a/src/main/java/net/minecraft/server/PlayerList.java b/src/main/java/net/minecraft/server/PlayerList.java
index 7a65b4f2cc6f3430bdad9df95b035ff29c8a2700..a212c3bb7c80948cc329f21c3916f52ec02e06bf 100644
--- a/src/main/java/net/minecraft/server/PlayerList.java
+++ b/src/main/java/net/minecraft/server/PlayerList.java
@@ -430,7 +430,7 @@ public abstract class PlayerList {
         cserver.getPluginManager().callEvent(playerQuitEvent);
         entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage());
 
-        entityplayer.playerTick(); // SPIGOT-924
+        if (server.isMainThread()) entityplayer.playerTick(); // SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog)
         // CraftBukkit end
 
         // Paper start - Remove from collideRule team if needed
diff --git a/src/main/java/net/minecraft/server/SystemUtils.java b/src/main/java/net/minecraft/server/SystemUtils.java
index 3299ac0b0115cb1166c1ee449bd49292e553b7af..dc7a4eabf83d5b74ccde170764dcd1e99f958ee0 100644
--- a/src/main/java/net/minecraft/server/SystemUtils.java
+++ b/src/main/java/net/minecraft/server/SystemUtils.java
@@ -119,6 +119,7 @@ public class SystemUtils {
         return SystemUtils.f;
     }
 
+    public static void shutdownServerThreadPool() { h(); } // Paper - OBFHELPER
     public static void h() {
         a(SystemUtils.e);
         a(SystemUtils.f);
diff --git a/src/main/java/net/minecraft/server/World.java b/src/main/java/net/minecraft/server/World.java
index d2b3e06dda10fc321768301c35f319767796d7d5..91294ff7e86a997cc81f287517b2ff5529fc0a91 100644
--- a/src/main/java/net/minecraft/server/World.java
+++ b/src/main/java/net/minecraft/server/World.java
@@ -840,6 +840,7 @@ public abstract class World implements GeneratorAccess, AutoCloseable {
 
                         gameprofilerfiller.exit();
                     } catch (Throwable throwable) {
+                        if (throwable instanceof ThreadDeath) throw throwable; // Paper
                         // Paper start - Prevent tile entity and entity crashes
                         String msg = "TileEntity threw exception at " + tileentity.world.getWorld().getName() + ":" + tileentity.position.getX() + "," + tileentity.position.getY() + "," + tileentity.position.getZ();
                         System.err.println(msg);
@@ -915,6 +916,7 @@ public abstract class World implements GeneratorAccess, AutoCloseable {
         try {
             consumer.accept(entity);
         } catch (Throwable throwable) {
+            if (throwable instanceof ThreadDeath) throw throwable; // Paper
             // Paper start - Prevent tile entity and entity crashes
             String msg = "Entity threw exception at " + entity.world.getWorld().getName() + ":" + entity.locX() + "," + entity.locY() + "," + entity.locZ();
             System.err.println(msg);
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
index 06b56bd65803bddc634d4984b8da29d910d31ee7..b0ab57ef2e45394905a29444daea1d7b3e866875 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
@@ -1761,7 +1761,7 @@ public final class CraftServer implements Server {
 
     @Override
     public boolean isPrimaryThread() {
-        return Thread.currentThread().equals(console.serverThread); // Paper - Fix issues with detecting main thread properly
+        return Thread.currentThread().equals(console.serverThread) || Thread.currentThread().equals(net.minecraft.server.MinecraftServer.getServer().shutdownThread); // Paper - Fix issues with detecting main thread properly, the only time Watchdog will be used is during a crash shutdown which is a "try our best" scenario
     }
 
     @Override
diff --git a/src/main/java/org/bukkit/craftbukkit/Main.java b/src/main/java/org/bukkit/craftbukkit/Main.java
index 438e86595760be233e89c49224963e5e8ce335e3..ff2b21305e4a46a8cafa247a0d4e786a6b7cb627 100644
--- a/src/main/java/org/bukkit/craftbukkit/Main.java
+++ b/src/main/java/org/bukkit/craftbukkit/Main.java
@@ -150,6 +150,37 @@ public class Main {
 
         OptionSet options = null;
 
+        // Paper start - preload logger classes to avoid plugins mixing versions
+        tryPreloadClass("com.destroystokyo.paper.log.LogFullPolicy");
+        tryPreloadClass("org.apache.logging.log4j.core.Core");
+        tryPreloadClass("org.apache.logging.log4j.core.Appender");
+        tryPreloadClass("org.apache.logging.log4j.core.ContextDataInjector");
+        tryPreloadClass("org.apache.logging.log4j.core.Filter");
+        tryPreloadClass("org.apache.logging.log4j.core.ErrorHandler");
+        tryPreloadClass("org.apache.logging.log4j.core.LogEvent");
+        tryPreloadClass("org.apache.logging.log4j.core.Logger");
+        tryPreloadClass("org.apache.logging.log4j.core.LoggerContext");
+        tryPreloadClass("org.apache.logging.log4j.core.LogEventListener");
+        tryPreloadClass("org.apache.logging.log4j.core.AbstractLogEvent");
+        tryPreloadClass("org.apache.logging.log4j.message.AsynchronouslyFormattable");
+        tryPreloadClass("org.apache.logging.log4j.message.FormattedMessage");
+        tryPreloadClass("org.apache.logging.log4j.message.ParameterizedMessage");
+        tryPreloadClass("org.apache.logging.log4j.message.Message");
+        tryPreloadClass("org.apache.logging.log4j.message.MessageFactory");
+        tryPreloadClass("org.apache.logging.log4j.message.TimestampMessage");
+        tryPreloadClass("org.apache.logging.log4j.message.SimpleMessage");
+        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLogger");
+        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerContext");
+        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncQueueFullPolicy");
+        tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerDisruptor");
+        tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEvent");
+        tryPreloadClass("org.apache.logging.log4j.core.async.DisruptorUtil");
+        tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEventHandler");
+        tryPreloadClass("org.apache.logging.log4j.core.impl.ThrowableProxy");
+        tryPreloadClass("org.apache.logging.log4j.core.impl.ThrowableProxy$CacheEntry");
+        tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedClassInfo");
+        tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedStackTraceElement");
+        // Paper end
         try {
             options = parser.parse(args);
         } catch (joptsimple.OptionException ex) {
@@ -245,8 +276,61 @@ public class Main {
             } catch (Throwable t) {
                 t.printStackTrace();
             }
+            // Paper start
+            // load some required classes to avoid errors during shutdown if jar is replaced
+            // also to guarantee our version loads over plugins
+            tryPreloadClass("com.destroystokyo.paper.util.SneakyThrow");
+            tryPreloadClass("com.google.common.collect.Iterators$PeekingImpl");
+            tryPreloadClass("com.google.common.collect.MapMakerInternalMap$Values");
+            tryPreloadClass("com.google.common.collect.MapMakerInternalMap$ValueIterator");
+            tryPreloadClass("com.google.common.collect.MapMakerInternalMap$WriteThroughEntry");
+            tryPreloadClass("com.google.common.collect.Iterables");
+            for (int i = 1; i <= 15; i++) {
+                tryPreloadClass("com.google.common.collect.Iterables$" + i, false);
+            }
+            tryPreloadClass("org.apache.commons.lang3.mutable.MutableBoolean");
+            tryPreloadClass("org.apache.commons.lang3.mutable.MutableInt");
+            tryPreloadClass("org.jline.terminal.impl.MouseSupport");
+            tryPreloadClass("org.jline.terminal.impl.MouseSupport$1");
+            tryPreloadClass("org.jline.terminal.Terminal$MouseTracking");
+            tryPreloadClass("co.aikar.timings.TimingHistory");
+            tryPreloadClass("co.aikar.timings.TimingHistory$MinuteReport");
+            tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext");
+            tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$11");
+            tryPreloadClass("io.netty.channel.AbstractChannel$AbstractUnsafe$8");
+            tryPreloadClass("io.netty.util.concurrent.DefaultPromise");
+            tryPreloadClass("io.netty.util.concurrent.DefaultPromise$1");
+            tryPreloadClass("io.netty.util.internal.PromiseNotificationUtil");
+            tryPreloadClass("io.netty.util.internal.SystemPropertyUtil");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$1");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$2");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$3");
+            tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$4");
+            tryPreloadClass("org.slf4j.helpers.MessageFormatter");
+            tryPreloadClass("org.slf4j.helpers.FormattingTuple");
+            tryPreloadClass("org.slf4j.helpers.BasicMarker");
+            tryPreloadClass("org.slf4j.helpers.Util");
+            tryPreloadClass("com.destroystokyo.paper.event.player.PlayerConnectionCloseEvent");
+            // Minecraft, seen during saving
+            tryPreloadClass("net.minecraft.server.LightEngineLayerEventListener$Void");
+            tryPreloadClass("net.minecraft.server.LightEngineLayerEventListener");
+            // Paper end
+        }
+    }
+
+    // Paper start
+    private static void tryPreloadClass(String className) {
+        tryPreloadClass(className, true);
+    }
+    private static void tryPreloadClass(String className, boolean printError) {
+        try {
+            Class.forName(className);
+        } catch (ClassNotFoundException e) {
+            if (printError) System.err.println("An expected class  " + className + " was not found for preloading: " + e.getMessage());
         }
     }
+    // Paper end
 
     private static List<String> asList(String... params) {
         return Arrays.asList(params);
diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
index 449e99d1b673870ed6892f6ab2c715a2db35c35d..c7ed6e0f8a989cec97700df2b15198c9c481c549 100644
--- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
+++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
@@ -12,12 +12,27 @@ public class ServerShutdownThread extends Thread {
     @Override
     public void run() {
         try {
+            // Paper start - try to shutdown on main
+            server.safeShutdown(false, false);
+            for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) {
+                Thread.sleep(100);
+            }
+            if (server.hasStopped()) {
+                while (!server.hasFullyShutdown) Thread.sleep(1000);
+                return;
+            }
+            // Looks stalled, close async
             org.spigotmc.AsyncCatcher.enabled = false; // Spigot
             org.spigotmc.AsyncCatcher.shuttingDown = true; // Paper
+            server.forceTicks = true;
             server.close();
+            while (!server.hasFullyShutdown) Thread.sleep(1000);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+            // Paper end
         } finally {
             try {
-                net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+                //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
             } catch (Exception e) {
             }
         }
diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java
index aefea3a9a8b9b75c62bd20018be7cd166a213001..123de5ac9026508e21cdc225f0962f5c3c46fed5 100644
--- a/src/main/java/org/spigotmc/RestartCommand.java
+++ b/src/main/java/org/spigotmc/RestartCommand.java
@@ -139,7 +139,7 @@ public class RestartCommand extends Command
     // Paper end
 
     // Paper start - copied from above and modified to return if the hook registered
-    private static boolean addShutdownHook(String restartScript)
+    public static boolean addShutdownHook(String restartScript)
     {
         String[] split = restartScript.split( " " );
         if ( split.length > 0 && new File( split[0] ).isFile() )
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
index 5bdcdcf9e85b73086722783bff26321d03382bb9..513c1041c34ebb3ac1775674a3f4526693759c08 100644
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
@@ -13,6 +13,7 @@ import org.bukkit.Bukkit;
 public class WatchdogThread extends Thread
 {
 
+    public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper
     private static WatchdogThread instance;
     private final long timeoutTime;
     private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
@@ -41,6 +42,7 @@ public class WatchdogThread extends Thread
     {
         if ( instance == null )
         {
+            if (timeoutTime <= 0) timeoutTime = 300; // Paper
             instance = new WatchdogThread( timeoutTime * 1000L, restart );
             instance.start();
         }
@@ -67,12 +69,13 @@ public class WatchdogThread extends Thread
             // Paper start
             Logger log = Bukkit.getServer().getLogger();
             long currentTime = monotonicMillis();
-            if ( lastTick != 0 && currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog") )
+            MinecraftServer server = MinecraftServer.getServer();
+            if (lastTick != 0 && hasStarted && (!server.isRunning() || (currentTime > lastTick + earlyWarningEvery && !DISABLE_WATCHDOG) ))
             {
-                boolean isLongTimeout = currentTime > lastTick + timeoutTime;
+                boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000);
                 // Don't spam early warning dumps
                 if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
-                if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+                if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
                 lastEarlyWarning = currentTime;
                 if (isLongTimeout) {
                 // Paper end
@@ -114,7 +117,7 @@ public class WatchdogThread extends Thread
                 log.log( Level.SEVERE, "------------------------------" );
                 log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
                 ChunkTaskManager.dumpAllChunkLoadInfo(); // Paper
-                dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
+                dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( server.serverThread.getId(), Integer.MAX_VALUE ), log );
                 log.log( Level.SEVERE, "------------------------------" );
                 //
                 // Paper start - Only print full dump on long timeouts
@@ -135,9 +138,24 @@ public class WatchdogThread extends Thread
 
                 if ( isLongTimeout )
                 {
-                if ( restart && !MinecraftServer.getServer().hasStopped() )
+                if ( !server.hasStopped() )
                 {
-                    RestartCommand.restart();
+                    AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us
+                    AsyncCatcher.shuttingDown = true;
+                    server.forceTicks = true;
+                    if (restart) {
+                        RestartCommand.addShutdownHook( SpigotConfig.restartScript );
+                    }
+                    // try one last chance to safe shutdown on main incase it 'comes back'
+                    server.safeShutdown(false, restart);
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
+                    if (!server.hasStopped()) {
+                        server.close();
+                    }
                 }
                 break;
                 } // Paper end
diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
index 476f4a5cbe664ddd05474cb88553018bd334a5b8..8af159abd3d0cc94cf155fec5b384c42f69551bf 100644
--- a/src/main/resources/log4j2.xml
+++ b/src/main/resources/log4j2.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="WARN" packages="com.mojang.util">
+<Configuration status="WARN" packages="com.mojang.util" shutdownHook="disable">
     <Appenders>
         <Queue name="ServerGuiConsole">
             <PatternLayout pattern="[%d{HH:mm:ss} %level]: %msg%n" />