Rewrite User storage and UUID cache

.gitignore

@@ -15,6 +15,8 @@
 /Essentials/kits.yml
 /Essentials/userdata/testplayer1.yml
 /Essentials/usermap.csv
+/Essentials/usermap.bin
+/Essentials/uuids.bin
 
 # Build files
 .gradle/

Essentials/src/main/java/com/earth2me/essentials/BalanceTopImpl.java

@@ -28,8 +28,8 @@ public BalanceTopImpl(IEssentials ess) {
     private void calculateBalanceTopMap() {
         final List<Entry> entries = new LinkedList<>();
         BigDecimal newTotal = BigDecimal.ZERO;
-        for (UUID u : ess.getUserMap().getAllUniqueUsers()) {
-            final User user = ess.getUserMap().getUser(u);
+        for (UUID u : ess.getUsers().getAllUserUUIDs()) {
+            final User user = ess.getUsers().loadUncachedUser(u);
             if (user != null) {
                 if (!ess.getSettings().isNpcsInBalanceRanking() && user.isNPC()) {
                     // Don't list NPCs in output

Essentials/src/main/java/com/earth2me/essentials/Essentials.java

@@ -39,6 +39,7 @@
 import com.earth2me.essentials.textreader.KeywordReplacer;
 import com.earth2me.essentials.textreader.SimpleTextInput;
 import com.earth2me.essentials.updatecheck.UpdateChecker;
+import com.earth2me.essentials.userstorage.ModernUserMap;
 import com.earth2me.essentials.utils.FormatUtil;
 import com.earth2me.essentials.utils.VersionUtil;
 import io.papermc.lib.PaperLib;
@@ -155,7 +156,9 @@ public class Essentials extends JavaPlugin implements net.ess3.api.IEssentials {
     private transient CustomItemResolver customItemResolver;
     private transient PermissionsHandler permissionsHandler;
     private transient AlternativeCommandsHandler alternativeCommandsHandler;
-    private transient UserMap userMap;
+    @Deprecated
+    private transient UserMap legacyUserMap;
+    private transient ModernUserMap userMap;
     private transient BalanceTopImpl balanceTop;
     private transient ExecuteTimer execTimer;
     private transient MailService mail;
@@ -222,7 +225,7 @@ public void setupForTesting(final Server server) throws IOException, InvalidDesc
         LOGGER.log(Level.INFO, dataFolder.toString());
         settings = new Settings(this);
         mail = new MailServiceImpl(this);
-        userMap = new UserMap(this);
+        userMap = new ModernUserMap(this);
         balanceTop = new BalanceTopImpl(this);
         permissionsHandler = new PermissionsHandler(this, false);
         Economy.setEss(this);
@@ -301,11 +304,14 @@ public void onEnable() {
             confList.add(settings);
             execTimer.mark("Settings");
 
+            upgrade.preModules();
+            execTimer.mark("Upgrade2");
+
             mail = new MailServiceImpl(this);
             execTimer.mark("Init(Mail)");
 
-            userMap = new UserMap(this);
-            confList.add(userMap);
+            userMap = new ModernUserMap(this);
+            legacyUserMap = new UserMap(userMap);
             execTimer.mark("Init(Usermap)");
 
             balanceTop = new BalanceTopImpl(this);
@@ -317,7 +323,7 @@ public void onEnable() {
             execTimer.mark("Kits");
 
             upgrade.afterSettings();
-            execTimer.mark("Upgrade2");
+            execTimer.mark("Upgrade3");
 
             warps = new Warps(this.getDataFolder());
             confList.add(warps);
@@ -588,7 +594,7 @@ public void onDisable() {
 
         Economy.setEss(null);
         Trade.closeLog();
-        getUserMap().getUUIDMap().shutdown();
+        getUsers().shutdown();
 
         HandlerList.unregisterAll(this);
     }
@@ -1067,7 +1073,7 @@ public User getUser(final Player base) {
             if (getSettings().isDebug()) {
                 LOGGER.log(Level.INFO, "Constructing new userfile from base player " + base.getName());
             }
-            user = new User(base, this);
+            user = userMap.loadUncachedUser(base);
         } else {
             user.update(base);
         }
@@ -1203,7 +1209,13 @@ public IItemDb getItemDb() {
     }
 
     @Override
+    @Deprecated
     public UserMap getUserMap() {
+        return legacyUserMap;
+    }
+
+    @Override
+    public ModernUserMap getUsers() {
         return userMap;
     }
 

Essentials/src/main/java/com/earth2me/essentials/EssentialsPlayerListener.java

@@ -308,6 +308,7 @@ public void delayedJoin(final Player player, final String message) {
 
         ess.getBackup().onPlayerJoin();
         final User dUser = ess.getUser(player);
+        dUser.update(player);
 
         dUser.startTransaction();
         if (dUser.isNPC()) {
@@ -367,7 +368,7 @@ public void run() {
                 } else if (ess.getSettings().isCustomJoinMessage()) {
                     final String msg = (newUsername ? ess.getSettings().getCustomNewUsernameMessage() : ess.getSettings().getCustomJoinMessage())
                         .replace("{PLAYER}", player.getDisplayName()).replace("{USERNAME}", player.getName())
-                        .replace("{UNIQUE}", NumberFormat.getInstance().format(ess.getUserMap().getUniqueUsers()))
+                        .replace("{UNIQUE}", NumberFormat.getInstance().format(ess.getUsers().getUserCount()))
                         .replace("{ONLINE}", NumberFormat.getInstance().format(ess.getOnlinePlayers().size()))
                         .replace("{UPTIME}", DateUtil.formatDateDiff(ManagementFactory.getRuntimeMXBean().getStartTime()))
                         .replace("{PREFIX}", FormatUtil.replaceFormat(ess.getPermissionsHandler().getPrefix(player)))
@@ -527,6 +528,7 @@ public void onPlayerLoginBanned(final PlayerLoginEvent event) {
     public void onPlayerLogin(final PlayerLoginEvent event) {
         if (event.getResult() == Result.KICK_FULL) {
             final User kfuser = ess.getUser(event.getPlayer());
+            kfuser.update(event.getPlayer());
             if (kfuser.isAuthorized("essentials.joinfullserver")) {
                 event.allow();
                 return;

Essentials/src/main/java/com/earth2me/essentials/EssentialsUpgrade.java

@@ -4,9 +4,9 @@
 import com.earth2me.essentials.config.EssentialsConfiguration;
 import com.earth2me.essentials.config.EssentialsUserConfiguration;
 import com.earth2me.essentials.craftbukkit.BanLookup;
+import com.earth2me.essentials.userstorage.ModernUUIDCache;
 import com.earth2me.essentials.utils.StringUtil;
 import com.google.common.base.Charsets;
-import com.google.common.collect.Maps;
 import com.google.common.io.Files;
 import com.google.gson.reflect.TypeToken;
 import net.ess3.api.IEssentials;
@@ -31,10 +31,10 @@
 import java.math.BigInteger;
 import java.security.DigestInputStream;
 import java.security.MessageDigest;
-import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
@@ -47,13 +47,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static com.earth2me.essentials.I18n.tl;
 
 public class EssentialsUpgrade {
-    private static final FileFilter YML_FILTER = pathname -> pathname.isFile() && pathname.getName().endsWith(".yml");
+    public static final FileFilter YML_FILTER = pathname -> pathname.isFile() && pathname.getName().endsWith(".yml");
     private static final String PATTERN_CONFIG_UUID_REGEX = "(?mi)^uuid:\\s*([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\\s*$";
     private static final Pattern PATTERN_CONFIG_UUID = Pattern.compile(PATTERN_CONFIG_UUID_REGEX);
     private static final String PATTERN_CONFIG_NAME_REGEX = "(?mi)^lastAccountName:\\s*[\"\']?(\\w+)[\"\']?\\s*$";
@@ -93,7 +92,6 @@ public static void uuidFileConvert(final IEssentials ess, final Boolean ignoreUF
             final int showProgress = countFiles % 250;
 
             if (showProgress == 0) {
-                ess.getUserMap().getUUIDMap().forceWriteUUIDMap();
                 ess.getLogger().info("Converted " + countFiles + "/" + userdir.list().length);
             }
 
@@ -103,7 +101,8 @@ public static void uuidFileConvert(final IEssentials ess, final Boolean ignoreUF
             final EssentialsUserConfiguration config;
             UUID uuid = null;
             try {
-                uuid = UUID.fromString(name);
+                //noinspection ResultOfMethodCallIgnored
+                UUID.fromString(name);
             } catch (final IllegalArgumentException ex) {
                 final File file = new File(userdir, string);
                 final EssentialsConfiguration conf = new EssentialsConfiguration(file);
@@ -115,38 +114,40 @@ public static void uuidFileConvert(final IEssentials ess, final Boolean ignoreUF
 
                 final String uuidString = conf.getString(uuidConf, null);
 
+                //noinspection ConstantConditions
                 for (int i = 0; i < 4; i++) {
                     try {
                         uuid = UUID.fromString(uuidString);
                         countEssCache++;
                         break;
                     } catch (final Exception ex2) {
                         if (conf.getBoolean("npc", false)) {
-                            uuid = UUID.nameUUIDFromBytes(("NPC:" + name).getBytes(Charsets.UTF_8));
+                            uuid = UUID.nameUUIDFromBytes(("NPC:" + (ess.getSettings().isSafeUsermap() ? StringUtil.safeString(name) : name)).getBytes(Charsets.UTF_8));
                             break;
                         }
 
                         final org.bukkit.OfflinePlayer player = ess.getServer().getOfflinePlayer(name);
                         uuid = player.getUniqueId();
                     }
 
+                    //noinspection ConstantConditions
                     if (uuid != null) {
                         countBukkit++;
                         break;
                     }
                 }
 
+                //noinspection ConstantConditions
                 if (uuid != null) {
                     conf.blockingSave();
                     config = new EssentialsUserConfiguration(name, uuid, new File(userdir, uuid + ".yml"));
                     config.convertLegacyFile();
-                    ess.getUserMap().trackUUID(uuid, name, false);
+                    ess.getUsers().loadUncachedUser(uuid);
                     continue;
                 }
                 countFails++;
             }
         }
-        ess.getUserMap().getUUIDMap().forceWriteUUIDMap();
 
         ess.getLogger().info("Converted " + countFiles + "/" + countFiles + ".  Conversion complete.");
         ess.getLogger().info("Converted via cache: " + countEssCache + " :: Converted via lookup: " + countBukkit + " :: Failed to convert: " + countFails);
@@ -913,74 +914,76 @@ private void updateBan(final String playerName, final String banReason, final Lo
         Bukkit.getBanList(BanList.Type.NAME).addBan(playerName, banReason, banTimeout == 0 ? null : new Date(banTimeout), Console.NAME);
     }
 
-    private void repairUserMap() {
-        if (doneFile.getBoolean("userMapRepaired", false)) {
+    public void generateUidCache() {
+        if (doneFile.getBoolean("newUidCacheBuilt", false)) {
             return;
         }
-        ess.getLogger().info("Starting usermap repair");
 
+        final File usermapFile = new File(ess.getDataFolder(), "usermap.bin");
+        final File uidsFile = new File(ess.getDataFolder(), "uuids.bin");
         final File userdataFolder = new File(ess.getDataFolder(), "userdata");
-        if (!userdataFolder.isDirectory()) {
+
+        if (!userdataFolder.isDirectory() || usermapFile.exists() || uidsFile.exists()) {
             ess.getLogger().warning("Missing userdata folder, aborting");
+            doneFile.setProperty("newUidCacheBuilt", true);
+            doneFile.save();
             return;
         }
-        final File[] files = userdataFolder.listFiles(YML_FILTER);
 
-        final DecimalFormat format = new DecimalFormat("#0.00");
-        final Map<String, UUID> names = Maps.newHashMap();
+        try {
+            if (!usermapFile.createNewFile() || !uidsFile.createNewFile()) {
+                ess.getLogger().warning("Couldn't create usermap.bin or uuids.bin, aborting");
+                return;
+            }
 
-        for (int index = 0; index < files.length; index++) {
-            final File file = files[index];
-            try {
-                UUID uuid = null;
-                final String filename = file.getName();
-                final String configData = new String(java.nio.file.Files.readAllBytes(file.toPath()), Charsets.UTF_8);
+            final Map<UUID, Long> uuids = new HashMap<>();
+            final Map<String, UUID> nameToUuidMap = new HashMap<>();
 
-                if (filename.length() > 36) {
+            final File[] files = userdataFolder.listFiles(YML_FILTER);
+            if (files != null) {
+                for (final File file : files) {
                     try {
-                        // ".yml" ending has 4 chars...
-                        uuid = UUID.fromString(filename.substring(0, filename.length() - 4));
-                    } catch (final IllegalArgumentException ignored) {
-                    }
-                }
-
-                final Matcher uuidMatcher = PATTERN_CONFIG_UUID.matcher(configData);
-                if (uuidMatcher.find()) {
-                    try {
-                        uuid = UUID.fromString(uuidMatcher.group(1));
-                    } catch (final IllegalArgumentException ignored) {
-                    }
-                }
-
-                if (uuid == null) {
-                    // Don't import
-                    continue;
-                }
-
-                final Matcher nameMatcher = PATTERN_CONFIG_NAME.matcher(configData);
-                if (nameMatcher.find()) {
-                    final String username = nameMatcher.group(1);
-                    if (username != null && username.length() > 0) {
-                        names.put(StringUtil.safeString(username), uuid);
+                        final String fileName = file.getName();
+                        final UUID uuid = UUID.fromString(fileName.substring(0, fileName.length() - 4));
+                        final EssentialsConfiguration config = new EssentialsConfiguration(file);
+                        config.load();
+                        String name = config.getString("last-account-name", null);
+                        name = ess.getSettings().isSafeUsermap() ? StringUtil.safeString(name) : name;
+                        final long time = config.getLong("timestamps.logout", 0L);
+
+                        if (name != null) {
+                            if (nameToUuidMap.containsKey(name)) {
+                                final UUID oldUuid = nameToUuidMap.get(name);
+                                if (oldUuid.version() < uuid.version() || (oldUuid.version() == uuid.version() && uuids.get(oldUuid) < time)) {
+                                    ess.getLogger().warning("New UUID found for " + name + ": " + uuid + " (old: " + oldUuid + "). Replacing.");
+                                    uuids.remove(oldUuid);
+                                } else {
+                                    ess.getLogger().warning("Found UUID for " + name + ": " + uuid + " (old: " + oldUuid + "). Skipping.");
+                                    continue;
+                                }
+                            }
+
+                            uuids.put(uuid, config.getLong("timestamps.logout", 0L));
+                            nameToUuidMap.put(name, uuid);
+                        }
+                    } catch (IllegalArgumentException | IndexOutOfBoundsException ignored) {
                     }
                 }
+            }
 
-                if (index % 1000 == 0) {
-                    ess.getLogger().info("Reading: " + format.format((100d * (double) index) / files.length)
-                        + "%");
-                }
-            } catch (final IOException e) {
-                ess.getLogger().log(Level.SEVERE, "Error while reading file: ", e);
-                return;
+            if (!nameToUuidMap.isEmpty()) {
+                ModernUUIDCache.writeNameUuidMap(usermapFile, nameToUuidMap);
             }
-        }
 
-        ess.getUserMap().getNames().putAll(names);
-        ess.getUserMap().reloadConfig();
+            if (!uuids.isEmpty()) {
+                ModernUUIDCache.writeUuidCache(uidsFile, uuids.keySet());
+            }
 
-        doneFile.setProperty("userMapRepaired", true);
-        doneFile.save();
-        ess.getLogger().info("Completed usermap repair.");
+            doneFile.setProperty("newUidCacheBuilt", true);
+            doneFile.save();
+        } catch (final IOException e) {
+            ess.getLogger().log(Level.SEVERE, "Error while generating initial uuids/names cache", e);
+        }
     }
 
     public void beforeSettings() {
@@ -991,6 +994,10 @@ public void beforeSettings() {
         moveMotdRulesToFile("rules");
     }
 
+    public void preModules() {
+        generateUidCache();
+    }
+
     public void afterSettings() {
         sanitizeAllUserFilenames();
         updateUsersPowerToolsFormat();
@@ -1001,7 +1008,6 @@ public void afterSettings() {
         uuidFileChange();
         banFormatChange();
         warnMetrics();
-        repairUserMap();
         convertIgnoreList();
         convertStupidCamelCaseUserdataKeys();
         convertMailList();