From c7836b80e470abce5ad06cb8da32d49233cb87db Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Fri, 30 Aug 2024 01:58:19 +0800 Subject: [PATCH 1/3] [chore] Reduce range of nextId when calling advanceNextId() --- .../src/main/java/org/apache/doris/catalog/Env.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 83907255641384..cdb2d8eb9bfdf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1679,10 +1679,13 @@ private void transferToMaster() { */ void advanceNextId() { long currentId = idGenerator.getBatchEndId(); - long currentNanos = System.nanoTime(); + long currentMill = System.currentTimeMillis(); long nextId = currentId + 1; - if (nextId < currentNanos) { - nextId = currentNanos; + // Reserve at most 8 billion for use in case of bugs or frequent reboots (21 billion reboots) + if ((1L << 63) - nextId < (1L << 33)) { + LOG.warn("nextId is too large: {}, it may be a bug and consider backup and migration", nextId); + } else { + nextId = (currentId + 1) < currentMill ? currentMill : currentId + (1L << 32); } // ATTN: Because MetaIdGenerator has guaranteed that each id it returns must have From 16a6b2aad87e966619e4fe365f50369e3d1bf4d1 Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Fri, 30 Aug 2024 11:33:08 +0800 Subject: [PATCH 2/3] Update fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java --- fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index cdb2d8eb9bfdf5..d7d5103be0413a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1685,6 +1685,8 @@ void advanceNextId() { if ((1L << 63) - nextId < (1L << 33)) { LOG.warn("nextId is too large: {}, it may be a bug and consider backup and migration", nextId); } else { + // Keep compatible with previous impl, the previous impl may result in extreme large nextId, + // and guess there are no more than 1L<<32 (~4e9) ids used since last reboot nextId = (currentId + 1) < currentMill ? currentMill : currentId + (1L << 32); } From b66748c40ead799378a4b79bc2148704e3df3fd7 Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Fri, 30 Aug 2024 11:59:27 +0800 Subject: [PATCH 3/3] Update fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java --- fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index d7d5103be0413a..9df93357f01256 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1681,8 +1681,8 @@ void advanceNextId() { long currentId = idGenerator.getBatchEndId(); long currentMill = System.currentTimeMillis(); long nextId = currentId + 1; - // Reserve at most 8 billion for use in case of bugs or frequent reboots (21 billion reboots) - if ((1L << 63) - nextId < (1L << 33)) { + // Reserve ~1 trillion for use in case of bugs or frequent reboots (~2 billion reboots) + if ((1L << 63) - nextId < (1L << 40)) { LOG.warn("nextId is too large: {}, it may be a bug and consider backup and migration", nextId); } else { // Keep compatible with previous impl, the previous impl may result in extreme large nextId,