diff --git a/Spigot-Server-Patches/0581-Lazily-track-plugin-scoreboards-by-default.patch b/Spigot-Server-Patches/0581-Lazily-track-plugin-scoreboards-by-default.patch
new file mode 100644
index 000000000..cec246524
--- /dev/null
+++ b/Spigot-Server-Patches/0581-Lazily-track-plugin-scoreboards-by-default.patch
@@ -0,0 +1,89 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Andrew Steinborn <git@steinborn.me>
+Date: Sat, 3 Oct 2020 04:15:09 -0400
+Subject: [PATCH] Lazily track plugin scoreboards by default
+
+On servers with plugins that constantly churn through scoreboards, there is a risk of
+degraded GC performance due to the number of scoreboards held on by weak references.
+Most plugins don't even need the (vanilla) functionality that requires all plugin
+scoreboards to be tracked by the server. Instead, only track scoreboards when an
+objective is added with a non-dummy criteria.
+
+This is a breaking change, however the change is a much more sensible default. In case
+this breaks your workflow you can always force all scoreboards to be tracked with
+settings.track-plugin-scoreboards in paper.yml.
+
+diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java
+index f65d3545039a2c471819ab5950c171ef6193cdb4..e14d71d60c3433c39dcf217d48bd667b2c2b49e1 100644
+--- a/src/main/java/com/destroystokyo/paper/PaperConfig.java
++++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java
+@@ -457,4 +457,9 @@ public class PaperConfig {
+     private static void maxJoinsPerTick() {
+         maxJoinsPerTick = getInt("settings.max-joins-per-tick", 3);
+     }
++
++    public static boolean trackPluginScoreboards;
++    private static void trackPluginScoreboards() {
++        trackPluginScoreboards = getBoolean("settings.track-plugin-scoreboards", false);
++    }
+ }
+diff --git a/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboard.java b/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboard.java
+index e3036fe23fa2be100044332c432d1ad5b4872823..b298fbd66b66c83e2ede6871025eaeecb409f28b 100644
+--- a/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboard.java
++++ b/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboard.java
+@@ -19,6 +19,7 @@ import org.bukkit.scoreboard.Team;
+ 
+ public final class CraftScoreboard implements org.bukkit.scoreboard.Scoreboard {
+     final Scoreboard board;
++    boolean registeredGlobally = false; // Paper
+ 
+     CraftScoreboard(Scoreboard board) {
+         this.board = board;
+@@ -45,6 +46,12 @@ public final class CraftScoreboard implements org.bukkit.scoreboard.Scoreboard {
+         Validate.isTrue(board.getObjective(name) == null, "An objective of name '" + name + "' already exists");
+ 
+         CraftCriteria craftCriteria = CraftCriteria.getFromBukkit(criteria);
++        // Paper start
++        if (craftCriteria.criteria != net.minecraft.server.IScoreboardCriteria.DUMMY && !registeredGlobally) {
++            net.minecraft.server.MinecraftServer.getServer().server.getScoreboardManager().registerScoreboardForVanilla(this);
++            registeredGlobally = true;
++        }
++        // Paper end
+         ScoreboardObjective objective = board.registerObjective(name, craftCriteria.criteria, CraftChatMessage.fromStringOrNull(displayName), CraftScoreboardTranslations.fromBukkitRender(renderType));
+         return new CraftObjective(this, objective);
+     }
+diff --git a/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboardManager.java b/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboardManager.java
+index ca2be30609159e6ca98b363d75cbc3ac550bca31..6fa2e271f7f01cd0bf247e2071fa33bd8c5c6cbe 100644
+--- a/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboardManager.java
++++ b/src/main/java/org/bukkit/craftbukkit/scoreboard/CraftScoreboardManager.java
+@@ -30,6 +30,7 @@ public final class CraftScoreboardManager implements ScoreboardManager {
+ 
+     public CraftScoreboardManager(MinecraftServer minecraftserver, net.minecraft.server.Scoreboard scoreboardServer) {
+         mainScoreboard = new CraftScoreboard(scoreboardServer);
++        mainScoreboard.registeredGlobally = true; // Paper
+         server = minecraftserver;
+         scoreboards.add(mainScoreboard);
+     }
+@@ -43,10 +44,22 @@ public final class CraftScoreboardManager implements ScoreboardManager {
+     public CraftScoreboard getNewScoreboard() {
+         org.spigotmc.AsyncCatcher.catchOp("scoreboard creation"); // Spigot
+         CraftScoreboard scoreboard = new CraftScoreboard(new ScoreboardServer(server));
++        // Paper start
++        if (com.destroystokyo.paper.PaperConfig.trackPluginScoreboards) {
++            scoreboard.registeredGlobally = true;
+         scoreboards.add(scoreboard);
++        }
++        // Paper end
+         return scoreboard;
+     }
+ 
++    // Paper start
++    public void registerScoreboardForVanilla(CraftScoreboard scoreboard) {
++        org.spigotmc.AsyncCatcher.catchOp("scoreboard registration");
++        scoreboards.add(scoreboard);
++    }
++    // Paper end
++
+     // CraftBukkit method
+     public CraftScoreboard getPlayerBoard(CraftPlayer player) {
+         CraftScoreboard board = playerBoards.get(player);