Skip to content

[opt](variable) force update some variable by variable version #41607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 1 commit into from
Nov 7, 2024
Merged

[opt](variable) force update some variable by variable version #41607

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1599,6 +1599,7 @@ private void transferToMaster() {
// Set initial root password if master FE first time launch.
auth.setInitialRootPassword(Config.initial_root_password);
} else {
VariableMgr.forceUpdateVariables();
if (journalVersion <= FeMetaVersion.VERSION_114) {
// if journal version is less than 114, which means it is upgraded from version before 2.0.
// When upgrading from 1.2 to 2.0,
Expand Down
6 changes: 4 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,8 +256,10 @@ private String genSingleSessionVariableDoc(SessionVariable sv, Field field, Lang
}
sb.append("\n\n");
}
sb.append(VAR_READ_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.READ_ONLY).append("`\n\n");
sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`").append(varAttr.flag() == VariableMgr.GLOBAL).append("`\n\n");
sb.append(VAR_READ_ONLY[lang.idx]).append("`")
.append((varAttr.flag() & VariableMgr.READ_ONLY) != 0).append("`\n\n");
sb.append(VAR_GLOBAL_ONLY[lang.idx]).append("`")
.append((varAttr.flag() & VariableMgr.GLOBAL) != 0).append("`\n\n");
return sb.toString();
}

Expand Down
10 changes: 10 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@
// NOTE: If you want access your variable safe, please hold VariableMgr's lock before access.
public final class GlobalVariable {

public static final int VARIABLE_VERSION_0 = 0;
public static final int VARIABLE_VERSION_100 = 100;
public static final int VARIABLE_VERSION_200 = 200;
public static final int VARIABLE_VERSION_300 = 300;
public static final int CURRENT_VARIABLE_VERSION = VARIABLE_VERSION_300;
public static final String VARIABLE_VERSION = "variable_version";

public static final String VERSION_COMMENT = "version_comment";
public static final String VERSION = "version";
public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names";
Expand Down Expand Up @@ -69,6 +76,9 @@ public final class GlobalVariable {

public static final String ENABLE_FETCH_ICEBERG_STATS = "enable_fetch_iceberg_stats";

@VariableMgr.VarAttr(name = VARIABLE_VERSION, flag = VariableMgr.INVISIBLE
| VariableMgr.READ_ONLY | VariableMgr.GLOBAL)
public static int variableVersion = CURRENT_VARIABLE_VERSION;

@VariableMgr.VarAttr(name = VERSION_COMMENT, flag = VariableMgr.READ_ONLY)
public static String versionComment = "Doris version "
Expand Down
3 changes: 1 addition & 2 deletions fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1486,7 +1486,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
@VariableMgr.VarAttr(name = ENABLE_NEREIDS_TIMEOUT, needForward = true)
public boolean enableNereidsTimeout = true;

@VariableMgr.VarAttr(name = "nereids_timeout_second", needForward = true)
@VariableMgr.VarAttr(name = NEREIDS_TIMEOUT_SECOND, needForward = true)
public int nereidsTimeoutSecond = 30;

@VariableMgr.VarAttr(name = ENABLE_PUSH_DOWN_NO_GROUP_AGG)
Expand Down Expand Up @@ -3887,7 +3887,6 @@ public void readFromJson(String json) throws IOException {
if (attr == null) {
continue;
}

if (!root.containsKey(attr.name())) {
continue;
}
Expand Down
55 changes: 54 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,11 @@ public static void replayGlobalVariableV2(GlobalVarPersistInfo info) throws DdlE
try {
String json = info.getPersistJsonString();
JSONObject root = (JSONObject) JSONValue.parse(json);
// if not variable version, we set it to 0 to ensure we could force set global variable.
boolean hasVariableVersion = root.containsKey(GlobalVariable.VARIABLE_VERSION);
if (!hasVariableVersion) {
GlobalVariable.variableVersion = GlobalVariable.VARIABLE_VERSION_0;
}
for (Object varName : root.keySet()) {
VarContext varContext = ctxByVarName.get((String) varName);
if (Env.isCheckpointThread()) {
Expand Down Expand Up @@ -728,11 +733,15 @@ public static List<List<String>> dump(SetType type, SessionVariable sessionVar,
rlock.lock();
try {
for (Map.Entry<String, VarContext> entry : ctxByDisplayVarName.entrySet()) {
// not show removed variables
VarAttr varAttr = entry.getValue().getField().getAnnotation(VarAttr.class);
// not show removed variables
if (VariableAnnotation.REMOVED.equals(varAttr.varType())) {
continue;
}
// not show invisible variables
if ((VariableMgr.INVISIBLE & varAttr.flag()) != 0) {
continue;
}
// Filter variable not match to the regex.
if (matcher != null && !matcher.match(entry.getKey())) {
continue;
Expand Down Expand Up @@ -947,4 +956,48 @@ private static ImmutableSortedMap.Builder<String, VarContext> getStringVarContex
}
return builder;
}

public static void forceUpdateVariables() {
int currentVariableVersion = GlobalVariable.variableVersion;
if (currentVariableVersion == GlobalVariable.VARIABLE_VERSION_0) {
// update from 2.0.15 or below to 2.0.16 or higher
if (VariableMgr.newSessionVariable().nereidsTimeoutSecond == 5) {
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.NEREIDS_TIMEOUT_SECOND, "30");
}
}
if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_100) {
// update from 2.1.6 or below to 2.1.7 or higher
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_NEREIDS_DML,
String.valueOf(true));
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_NEREIDS_DML_WITH_PIPELINE,
String.valueOf(true));
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_NEREIDS_PLANNER,
String.valueOf(true));
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER,
String.valueOf(true));
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_PIPELINE_X_ENGINE,
String.valueOf(true));
}
if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_200) {
// update from 3.0.2 or below to 3.0.3 or higher
VariableMgr.refreshDefaultSessionVariables("update variable version",
SessionVariable.ENABLE_FALLBACK_TO_ORIGINAL_PLANNER,
String.valueOf(false));
}
if (currentVariableVersion < GlobalVariable.VARIABLE_VERSION_300) {
// update to master
// do nothing
}
if (currentVariableVersion < GlobalVariable.CURRENT_VARIABLE_VERSION) {
VariableMgr.refreshDefaultSessionVariables("update variable version",
GlobalVariable.VARIABLE_VERSION,
String.valueOf(GlobalVariable.CURRENT_VARIABLE_VERSION));
}
}
}