From 56d54b9b4be9bbca62c06060f00ade0b66b6d160 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Thu, 19 Sep 2024 20:09:40 +0800 Subject: [PATCH 01/14] N --- .../apache/doris/common/profile/Profile.java | 107 +++++++++++------- .../apache/doris/common/util/DebugUtil.java | 34 ++++++ .../org/apache/doris/qe/StmtExecutor.java | 2 + .../java/org/apache/doris/qe/VariableMgr.java | 31 +++++ 4 files changed, 135 insertions(+), 39 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java index 88fd317879451e..2e313ed8a130c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java @@ -116,6 +116,8 @@ public class Profile { public Map rowsProducedMap = new HashMap<>(); private List physicalRelations = new ArrayList<>(); + private String changedSessionVarCache = ""; + // Need default constructor for read from storage public Profile() {} @@ -318,8 +320,9 @@ public String getProfileByLevel() { StringBuilder builder = new StringBuilder(); // add summary to builder summaryProfile.prettyPrint(builder); - // read execution profile from storage or generate it from memory (during query execution) + getChangedSessionVars(builder); getExecutionProfileContent(builder); + getOnStorageProfile(builder); return builder.toString(); } @@ -362,50 +365,14 @@ public String getProfileBrief() { Gson gson = new GsonBuilder().setPrettyPrinting().create(); return gson.toJson(rootProfile.toBrief()); } - - // Read file if profile has been stored to storage. + + // Return is profile has been stored to storage public void getExecutionProfileContent(StringBuilder builder) { if (builder == null) { builder = new StringBuilder(); } if (profileHasBeenStored()) { - LOG.info("Profile {} has been stored to storage, reading it from storage", id); - - FileInputStream fileInputStream = null; - - try { - fileInputStream = createPorfileFileInputStream(profileStoragePath); - if (fileInputStream == null) { - builder.append("Failed to read execution profile from " + profileStoragePath); - return; - } - - DataInputStream dataInput = new DataInputStream(fileInputStream); - // skip summary profile - Text.readString(dataInput); - // read compressed execution profile - int binarySize = dataInput.readInt(); - byte[] binaryExecutionProfile = new byte[binarySize]; - dataInput.readFully(binaryExecutionProfile, 0, binarySize); - // decompress binary execution profile - String textExecutionProfile = decompressExecutionProfile(binaryExecutionProfile); - builder.append(textExecutionProfile); - return; - } catch (Exception e) { - LOG.error("An error occurred while reading execution profile from storage, profile storage path: {}", - profileStoragePath, e); - builder.append("Failed to read execution profile from " + profileStoragePath); - } finally { - if (fileInputStream != null) { - try { - fileInputStream.close(); - } catch (Exception e) { - LOG.warn("Close profile {} failed", profileStoragePath, e); - } - } - } - return; } @@ -603,6 +570,7 @@ public void writeToStorage(String systemProfileStorageDir) { // store execution profiles as string StringBuilder build = new StringBuilder(); + getChangedSessionVars(build); getExecutionProfileContent(build); byte[] buf = compressExecutionProfile(build.toString()); dataOutputStream.writeInt(buf.length); @@ -692,4 +660,65 @@ private void updateActualRowCountOnPhysicalPlan(Plan plan) { updateActualRowCountOnPhysicalPlan(child); } } + + public void setChangedSessionVar(String changedSessionVar) { + this.changedSessionVarCache = changedSessionVar; + } + + private void getChangedSessionVars(StringBuilder builder) { + if (builder == null) { + builder = new StringBuilder(); + } + if (profileHasBeenStored()) { + return; + } + + builder.append("\nChanged Session Variables:\n"); + builder.append(changedSessionVarCache); + builder.append("\n"); + } + + private void getOnStorageProfile(StringBuilder builder) { + if (!profileHasBeenStored()) { + return; + } + + LOG.info("Profile {} has been stored to storage, reading it from storage", id); + + FileInputStream fileInputStream = null; + + try { + fileInputStream = createPorfileFileInputStream(profileStoragePath); + if (fileInputStream == null) { + builder.append("Failed to read execution profile from " + profileStoragePath); + return; + } + + DataInputStream dataInput = new DataInputStream(fileInputStream); + // skip summary profile + Text.readString(dataInput); + // read compressed execution profile + int binarySize = dataInput.readInt(); + byte[] binaryExecutionProfile = new byte[binarySize]; + dataInput.readFully(binaryExecutionProfile, 0, binarySize); + // decompress binary execution profile + String textExecutionProfile = decompressExecutionProfile(binaryExecutionProfile); + builder.append(textExecutionProfile); + return; + } catch (Exception e) { + LOG.error("An error occurred while reading execution profile from storage, profile storage path: {}", + profileStoragePath, e); + builder.append("Failed to read execution profile from " + profileStoragePath); + } finally { + if (fileInputStream != null) { + try { + fileInputStream.close(); + } catch (Exception e) { + LOG.warn("Close profile {} failed", profileStoragePath, e); + } + } + } + + return; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java index 937c74cac66bb5..6f111f4ad87ac8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java @@ -26,6 +26,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.text.DecimalFormat; +import java.util.List; import java.util.UUID; public class DebugUtil { @@ -177,4 +178,37 @@ public static String getStackTrace(Exception e) { e.printStackTrace(new PrintWriter(sw)); return sw.toString(); } + + public static String prettyPrintChangedSessionVar(List> nestedList) { + if (nestedList == null || nestedList.isEmpty()) { + return ""; + } + + StringBuilder output = new StringBuilder(); + + // Get the maximum width of each column for alignment + int[] columnWidths = new int[3]; // Assuming each inner list has exactly 3 columns + + // Iterate through the list to determine the maximum width of each column + for (List row : nestedList) { + for (int i = 0; i < row.size(); i++) { + columnWidths[i] = Math.max(columnWidths[i], row.get(i).length()); + } + } + + // Build the table with proper formatting and alignment + for (List row : nestedList) { + for (int i = 0; i < row.size(); i++) { + // Append each element, padded to the correct width, followed by " | " + output.append(String.format("%-" + columnWidths[i] + "s", row.get(i))); + if (i < row.size() - 1) { + output.append(" | "); // Separator between columns + } + } + output.append("\n"); // Newline after each row + } + + return output.toString(); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 402bb2b9661b57..ddcb5ed54997ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -671,6 +671,8 @@ private void executeByNereids(TUniqueId queryId) throws Exception { context.setQueryId(queryId); context.setStartTime(); profile.getSummaryProfile().setQueryBeginTime(); + List> changedSessionVar = VariableMgr.dumpChangedVars(context.getSessionVariable()); + profile.setChangedSessionVar(DebugUtil.prettyPrintChangedSessionVar(changedSessionVar)); context.setStmtId(STMT_ID_GENERATOR.incrementAndGet()); parseByNereids(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 1b1a317f5f0bb5..e2751f5f7ea5e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -31,6 +31,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.VariableAnnotation; +import org.apache.doris.common.util.DebugUtil; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.persist.GlobalVarPersistInfo; @@ -800,6 +801,36 @@ public int compare(List o1, List o2) { return changedRows; } + public static List> dumpChangedVars(SessionVariable sessionVar) { + List> changedRows = Lists.newArrayList(); + for (Map.Entry entry : ctxByDisplayVarName.entrySet()) { + VarContext ctx = entry.getValue(); + List row = Lists.newArrayList(); + String varName = entry.getKey(); + String curValue = getValue(sessionVar, ctx.getField()); + String defaultValue = ctx.getDefaultValue(); + if (VariableVarConverters.hasConverter(varName)) { + try { + defaultValue = VariableVarConverters.decode(varName, Long.valueOf(defaultValue)); + curValue = VariableVarConverters.decode(varName, Long.valueOf(curValue)); + } catch (DdlException e) { + row.add(""); + LOG.warn("Encode session variable failed"); + } + } + + if (curValue.equals(defaultValue)) { + continue; + } + + row.add(varName); + row.add(curValue); + row.add(defaultValue); + changedRows.add(row); + } + return changedRows; + } + @Retention(RetentionPolicy.RUNTIME) public @interface VarAttr { // Name in show variables and set statement; From 0cdecb557a2c63a2139176e06d1e5ceb157c135d Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Thu, 19 Sep 2024 20:45:44 +0800 Subject: [PATCH 02/14] X --- .../apache/doris/common/util/DebugUtil.java | 30 +++++-------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java index 6f111f4ad87ac8..7fe572c790d30c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java @@ -183,32 +183,16 @@ public static String prettyPrintChangedSessionVar(List> nestedList) if (nestedList == null || nestedList.isEmpty()) { return ""; } - + StringBuilder output = new StringBuilder(); - - // Get the maximum width of each column for alignment - int[] columnWidths = new int[3]; // Assuming each inner list has exactly 3 columns - - // Iterate through the list to determine the maximum width of each column - for (List row : nestedList) { - for (int i = 0; i < row.size(); i++) { - columnWidths[i] = Math.max(columnWidths[i], row.get(i).length()); - } - } - - // Build the table with proper formatting and alignment + output.append("\n"); + + // Iterate through the data rows for (List row : nestedList) { - for (int i = 0; i < row.size(); i++) { - // Append each element, padded to the correct width, followed by " | " - output.append(String.format("%-" + columnWidths[i] + "s", row.get(i))); - if (i < row.size() - 1) { - output.append(" | "); // Separator between columns - } - } + output.append(String.join("\t", row)); // Join row elements with tabs output.append("\n"); // Newline after each row } - + return output.toString(); - } - + } } From 844889c5012340e46ebd43f6bf9b88edb86b112e Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Thu, 19 Sep 2024 21:07:46 +0800 Subject: [PATCH 03/14] X --- .../apache/doris/common/util/DebugUtil.java | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java index 7fe572c790d30c..943a5253f7f766 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java @@ -185,11 +185,52 @@ public static String prettyPrintChangedSessionVar(List> nestedList) } StringBuilder output = new StringBuilder(); - output.append("\n"); - // Iterate through the data rows + // Assuming each inner list has exactly 3 columns + int[] columnWidths = new int[3]; + + // Calculate the maximum width of each column + // First consider the header widths: "VarName", "CurrentValue", "DefaultValue" + String[] headers = {"VarName", "CurrentValue", "DefaultValue"}; + for (int i = 0; i < headers.length; i++) { + columnWidths[i] = headers[i].length(); // Initialize with header length + } + + // Update column widths based on data + for (List row : nestedList) { + for (int i = 0; i < row.size(); i++) { + columnWidths[i] = Math.max(columnWidths[i], row.get(i).length()); + } + } + + // Build the table header + for (int i = 0; i < headers.length; i++) { + output.append(String.format("%-" + columnWidths[i] + "s", headers[i])); + if (i < headers.length - 1) { + output.append(" | "); // Separator between columns + } + } + output.append("\n"); // Newline after the header + + // Add a separator line for better readability (optional) + for (int i = 0; i < headers.length; i++) { + output.append(String.format("%-" + columnWidths[i] + "s", "-".repeat(columnWidths[i]))); + if (i < headers.length - 1) { + output.append("-|-"); // Separator between columns + } + } + output.append("\n"); // Newline after the separator + + // Build the table body with proper alignment based on column widths for (List row : nestedList) { - output.append(String.join("\t", row)); // Join row elements with tabs + for (int i = 0; i < row.size(); i++) { + String element = row.get(i); + // Pad with spaces if the element is shorter than the column width + output.append(String.format("%-" + columnWidths[i] + "s", element)); + if (i < row.size() - 1) { + output.append(" | "); // Separator between columns + } + } output.append("\n"); // Newline after each row } From fed692c3d7569c7f5975b210ace376d8c986a872 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Thu, 19 Sep 2024 21:12:34 +0800 Subject: [PATCH 04/14] FMT --- .../apache/doris/common/util/DebugUtil.java | 18 +++++++++--------- .../java/org/apache/doris/qe/VariableMgr.java | 1 - 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java index 943a5253f7f766..38250c64c0385a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java @@ -183,26 +183,26 @@ public static String prettyPrintChangedSessionVar(List> nestedList) if (nestedList == null || nestedList.isEmpty()) { return ""; } - + StringBuilder output = new StringBuilder(); - + // Assuming each inner list has exactly 3 columns int[] columnWidths = new int[3]; - + // Calculate the maximum width of each column // First consider the header widths: "VarName", "CurrentValue", "DefaultValue" String[] headers = {"VarName", "CurrentValue", "DefaultValue"}; for (int i = 0; i < headers.length; i++) { columnWidths[i] = headers[i].length(); // Initialize with header length } - + // Update column widths based on data for (List row : nestedList) { for (int i = 0; i < row.size(); i++) { columnWidths[i] = Math.max(columnWidths[i], row.get(i).length()); } } - + // Build the table header for (int i = 0; i < headers.length; i++) { output.append(String.format("%-" + columnWidths[i] + "s", headers[i])); @@ -211,7 +211,7 @@ public static String prettyPrintChangedSessionVar(List> nestedList) } } output.append("\n"); // Newline after the header - + // Add a separator line for better readability (optional) for (int i = 0; i < headers.length; i++) { output.append(String.format("%-" + columnWidths[i] + "s", "-".repeat(columnWidths[i]))); @@ -220,7 +220,7 @@ public static String prettyPrintChangedSessionVar(List> nestedList) } } output.append("\n"); // Newline after the separator - + // Build the table body with proper alignment based on column widths for (List row : nestedList) { for (int i = 0; i < row.size(); i++) { @@ -233,7 +233,7 @@ public static String prettyPrintChangedSessionVar(List> nestedList) } output.append("\n"); // Newline after each row } - + return output.toString(); - } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index e2751f5f7ea5e5..0e10fc290f13dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -31,7 +31,6 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.VariableAnnotation; -import org.apache.doris.common.util.DebugUtil; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.persist.GlobalVarPersistInfo; From dd5fb79ea61a22178849d3f69c36e990cd91dc0d Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Thu, 19 Sep 2024 21:18:31 +0800 Subject: [PATCH 05/14] X --- .../src/main/java/org/apache/doris/common/profile/Profile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java index 2e313ed8a130c3..d1b449c234b7f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java @@ -365,7 +365,7 @@ public String getProfileBrief() { Gson gson = new GsonBuilder().setPrettyPrinting().create(); return gson.toJson(rootProfile.toBrief()); } - + // Return is profile has been stored to storage public void getExecutionProfileContent(StringBuilder builder) { if (builder == null) { From 5e8214cccb20d6662f2b2e772c3c2b3365092130 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 20 Sep 2024 10:43:01 +0800 Subject: [PATCH 06/14] TEST --- .../apache/doris/common/profile/Profile.java | 2 +- .../print_changed_session_var.groovy | 90 +++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/query_profile/print_changed_session_var.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java index d1b449c234b7f5..0a65ecf7087e25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java @@ -366,7 +366,7 @@ public String getProfileBrief() { return gson.toJson(rootProfile.toBrief()); } - // Return is profile has been stored to storage + // Return if profile has been stored to storage public void getExecutionProfileContent(StringBuilder builder) { if (builder == null) { builder = new StringBuilder(); diff --git a/regression-test/suites/query_profile/print_changed_session_var.groovy b/regression-test/suites/query_profile/print_changed_session_var.groovy new file mode 100644 index 00000000000000..69cd67f5f1f7f3 --- /dev/null +++ b/regression-test/suites/query_profile/print_changed_session_var.groovy @@ -0,0 +1,90 @@ +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.json.StringEscapeUtils + +def getProfileList = { + def dst = 'http://' + context.config.feHttpAddress + def conn = new URL(dst + "/rest/v1/query_profile").openConnection() + conn.setRequestMethod("GET") + def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + + (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encoding}") + return conn.getInputStream().getText() +} + + +def getProfile = { id -> + def dst = 'http://' + context.config.feHttpAddress + def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection() + conn.setRequestMethod("GET") + def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + + (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encoding}") + return conn.getInputStream().getText() +} + + +suite('print_changed_session_var') { + sql """ + DROP TABLE IF EXISTS print_changed_session_var; + """ + sql """ + CREATE TABLE if not exists `print_changed_session_var` ( + `id` INT, + `name` varchar(32) + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + // Insert data to table + sql """ + insert into print_changed_session_var values + (1, "A"),(2, "B"),(3, "C"),(4, "D"),(5,"E"),(6,"F"),(7,"G"),(8,"H"),(9,"K"); + """ + sql """ + insert into print_changed_session_var values + (10, "A"),(20, "B"),(30, "C"),(40, "D"),(50,"E"),(60,"F"),(70,"G"),(80,"H"),(90,"K"); + """ + sql """ + insert into print_changed_session_var values + (101, "A"),(201, "B"),(301, "C"),(401, "D"),(501,"E"),(601,"F"),(701,"G"),(801,"H"),(901,"K"); + """ + sql """ + insert into print_changed_session_var values + (1010, "A"),(2010, "B"),(3010, "C"),(4010, "D"),(5010,"E"),(6010,"F"),(7010,"G"),(8010,"H"),(9010,"K"); + """ + + def uuidString = UUID.randomUUID().toString() + sql "set enable_profile=true" + sql "set parallel_pipeline_task_num=8" + sql """ + select "test_${uuidString}", * from print_changed_session_var limit 10; + """ + + def wholeString = getProfileList() + List profileData = new JsonSlurper().parseText(wholeString).data.rows + String queryId = ""; + + logger.info("Test query with id {}", uuidString) + + for (def profileItem in profileData) { + if (profileItem["Sql Statement"].toString().contains("test_${uuidString}")) { + queryId = profileItem["Profile ID"].toString() + logger.info("profileItem: {}", profileItem) + } + } + + logger.info("queryId_${uuidString}: {}", queryId) + + assertTrue(queryId != "") + + // Sleep 5 seconds to make sure profile collection is done + Thread.sleep(5000) + + def String profileContent = getProfile(queryId).toString() + logger.info("query {} profile\n{}", queryId, profileContent) + assertTrue(profileContent.contains("parallel_pipeline_task_num | 8")) +} \ No newline at end of file From 24b312544e25bb23f7939f01793ddd91e927b4c8 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 20 Sep 2024 10:46:27 +0800 Subject: [PATCH 07/14] LOCK --- .../java/org/apache/doris/qe/VariableMgr.java | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 0e10fc290f13dc..b30f84d68ce926 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -801,32 +801,39 @@ public int compare(List o1, List o2) { } public static List> dumpChangedVars(SessionVariable sessionVar) { + // Hold the read lock when session dump, because this option need to access global variable. + rlock.lock(); List> changedRows = Lists.newArrayList(); - for (Map.Entry entry : ctxByDisplayVarName.entrySet()) { - VarContext ctx = entry.getValue(); - List row = Lists.newArrayList(); - String varName = entry.getKey(); - String curValue = getValue(sessionVar, ctx.getField()); - String defaultValue = ctx.getDefaultValue(); - if (VariableVarConverters.hasConverter(varName)) { - try { - defaultValue = VariableVarConverters.decode(varName, Long.valueOf(defaultValue)); - curValue = VariableVarConverters.decode(varName, Long.valueOf(curValue)); - } catch (DdlException e) { - row.add(""); - LOG.warn("Encode session variable failed"); + try { + for (Map.Entry entry : ctxByDisplayVarName.entrySet()) { + VarContext ctx = entry.getValue(); + List row = Lists.newArrayList(); + String varName = entry.getKey(); + String curValue = getValue(sessionVar, ctx.getField()); + String defaultValue = ctx.getDefaultValue(); + if (VariableVarConverters.hasConverter(varName)) { + try { + defaultValue = VariableVarConverters.decode(varName, Long.valueOf(defaultValue)); + curValue = VariableVarConverters.decode(varName, Long.valueOf(curValue)); + } catch (DdlException e) { + row.add(""); + LOG.warn("Encode session variable failed"); + } } + + if (curValue.equals(defaultValue)) { + continue; + } + + row.add(varName); + row.add(curValue); + row.add(defaultValue); + changedRows.add(row); } - - if (curValue.equals(defaultValue)) { - continue; - } - - row.add(varName); - row.add(curValue); - row.add(defaultValue); - changedRows.add(row); + } finally { + rlock.unlock(); } + return changedRows; } From 85a8a6b02ccb79a6ee0bc32db5e609c1e9918894 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 20 Sep 2024 10:49:55 +0800 Subject: [PATCH 08/14] LICENSE --- .../print_changed_session_var.groovy | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/regression-test/suites/query_profile/print_changed_session_var.groovy b/regression-test/suites/query_profile/print_changed_session_var.groovy index 69cd67f5f1f7f3..383c82ef37dbf0 100644 --- a/regression-test/suites/query_profile/print_changed_session_var.groovy +++ b/regression-test/suites/query_profile/print_changed_session_var.groovy @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + import groovy.json.JsonOutput import groovy.json.JsonSlurper import groovy.json.StringEscapeUtils From 10c0fc2cf601f7a3d0fe5b11d3ada32ec654c461 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 20 Sep 2024 10:52:04 +0800 Subject: [PATCH 09/14] REFINE --- .../src/main/java/org/apache/doris/common/util/DebugUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java index 38250c64c0385a..39c4cd91904775 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DebugUtil.java @@ -214,7 +214,7 @@ public static String prettyPrintChangedSessionVar(List> nestedList) // Add a separator line for better readability (optional) for (int i = 0; i < headers.length; i++) { - output.append(String.format("%-" + columnWidths[i] + "s", "-".repeat(columnWidths[i]))); + output.append(String.format("%-" + columnWidths[i] + "s", Strings.repeat("-", columnWidths[i]))); if (i < headers.length - 1) { output.append("-|-"); // Separator between columns } From e98f13dacd0094b995140a9d7d58bf081b88a799 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 20 Sep 2024 11:38:44 +0800 Subject: [PATCH 10/14] FMT --- .../src/main/java/org/apache/doris/qe/VariableMgr.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index b30f84d68ce926..645a4f80090ee9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -820,11 +820,11 @@ public static List> dumpChangedVars(SessionVariable sessionVar) { LOG.warn("Encode session variable failed"); } } - + if (curValue.equals(defaultValue)) { continue; } - + row.add(varName); row.add(curValue); row.add(defaultValue); @@ -833,7 +833,7 @@ public static List> dumpChangedVars(SessionVariable sessionVar) { } finally { rlock.unlock(); } - + return changedRows; } From 440991b7f54aa51d92beed5b2c1e91e17ecab003 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 20 Sep 2024 16:45:19 +0800 Subject: [PATCH 11/14] REFINE --- fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 645a4f80090ee9..8bb4210c605607 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -816,8 +816,7 @@ public static List> dumpChangedVars(SessionVariable sessionVar) { defaultValue = VariableVarConverters.decode(varName, Long.valueOf(defaultValue)); curValue = VariableVarConverters.decode(varName, Long.valueOf(curValue)); } catch (DdlException e) { - row.add(""); - LOG.warn("Encode session variable failed"); + LOG.warn("Decode session variable {} failed, reason: {}", varName, e.getMessage()); } } From 9994508a53a5b59d4448551935c98e6718f09ad0 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 20 Sep 2024 19:05:53 +0800 Subject: [PATCH 12/14] X --- .../suites/query_profile/print_changed_session_var.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/query_profile/print_changed_session_var.groovy b/regression-test/suites/query_profile/print_changed_session_var.groovy index 383c82ef37dbf0..9befbc59ea5078 100644 --- a/regression-test/suites/query_profile/print_changed_session_var.groovy +++ b/regression-test/suites/query_profile/print_changed_session_var.groovy @@ -103,5 +103,5 @@ suite('print_changed_session_var') { def String profileContent = getProfile(queryId).toString() logger.info("query {} profile\n{}", queryId, profileContent) - assertTrue(profileContent.contains("parallel_pipeline_task_num | 8")) + assertTrue(profileContent.contains("parallel_pipeline_task_num | 8")) } \ No newline at end of file From daf7d7769750c96b36874968f7d3d9ebc077f64d Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Fri, 20 Sep 2024 19:42:36 +0800 Subject: [PATCH 13/14] NE" --- .../src/main/java/org/apache/doris/common/profile/Profile.java | 3 ++- .../main/java/org/apache/doris/common/util/ProfileManager.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java index 0a65ecf7087e25..c35ffdbe327db5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java @@ -440,8 +440,9 @@ public void setSummaryProfile(SummaryProfile summaryProfile) { this.summaryProfile = summaryProfile; } - public void releaseExecutionProfile() { + public void releaseMemory() { this.executionProfiles.clear(); + this.changedSessionVarCache = ""; } public boolean shouldStoreToStorage() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java index 65ddab151e8145..c6fc3307fe0b29 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java @@ -687,7 +687,7 @@ private void writeProfileToStorage() { for (ExecutionProfile executionProfile : profileElement.profile.getExecutionProfiles()) { this.queryIdToExecutionProfiles.remove(executionProfile.getQueryId()); } - profileElement.profile.releaseExecutionProfile(); + profileElement.profile.releaseMemory(); } } finally { writeLock.unlock(); From be59ba4cebba90393da310f79bc999412c8ad750 Mon Sep 17 00:00:00 2001 From: zhiqiang-hhhh Date: Sat, 21 Sep 2024 14:15:26 +0800 Subject: [PATCH 14/14] N --- .../print_changed_session_var.groovy | 107 ------------------ 1 file changed, 107 deletions(-) delete mode 100644 regression-test/suites/query_profile/print_changed_session_var.groovy diff --git a/regression-test/suites/query_profile/print_changed_session_var.groovy b/regression-test/suites/query_profile/print_changed_session_var.groovy deleted file mode 100644 index 9befbc59ea5078..00000000000000 --- a/regression-test/suites/query_profile/print_changed_session_var.groovy +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import groovy.json.StringEscapeUtils - -def getProfileList = { - def dst = 'http://' + context.config.feHttpAddress - def conn = new URL(dst + "/rest/v1/query_profile").openConnection() - conn.setRequestMethod("GET") - def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + - (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) - conn.setRequestProperty("Authorization", "Basic ${encoding}") - return conn.getInputStream().getText() -} - - -def getProfile = { id -> - def dst = 'http://' + context.config.feHttpAddress - def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection() - conn.setRequestMethod("GET") - def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + - (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) - conn.setRequestProperty("Authorization", "Basic ${encoding}") - return conn.getInputStream().getText() -} - - -suite('print_changed_session_var') { - sql """ - DROP TABLE IF EXISTS print_changed_session_var; - """ - sql """ - CREATE TABLE if not exists `print_changed_session_var` ( - `id` INT, - `name` varchar(32) - ) ENGINE=OLAP - DISTRIBUTED BY HASH(`id`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - - // Insert data to table - sql """ - insert into print_changed_session_var values - (1, "A"),(2, "B"),(3, "C"),(4, "D"),(5,"E"),(6,"F"),(7,"G"),(8,"H"),(9,"K"); - """ - sql """ - insert into print_changed_session_var values - (10, "A"),(20, "B"),(30, "C"),(40, "D"),(50,"E"),(60,"F"),(70,"G"),(80,"H"),(90,"K"); - """ - sql """ - insert into print_changed_session_var values - (101, "A"),(201, "B"),(301, "C"),(401, "D"),(501,"E"),(601,"F"),(701,"G"),(801,"H"),(901,"K"); - """ - sql """ - insert into print_changed_session_var values - (1010, "A"),(2010, "B"),(3010, "C"),(4010, "D"),(5010,"E"),(6010,"F"),(7010,"G"),(8010,"H"),(9010,"K"); - """ - - def uuidString = UUID.randomUUID().toString() - sql "set enable_profile=true" - sql "set parallel_pipeline_task_num=8" - sql """ - select "test_${uuidString}", * from print_changed_session_var limit 10; - """ - - def wholeString = getProfileList() - List profileData = new JsonSlurper().parseText(wholeString).data.rows - String queryId = ""; - - logger.info("Test query with id {}", uuidString) - - for (def profileItem in profileData) { - if (profileItem["Sql Statement"].toString().contains("test_${uuidString}")) { - queryId = profileItem["Profile ID"].toString() - logger.info("profileItem: {}", profileItem) - } - } - - logger.info("queryId_${uuidString}: {}", queryId) - - assertTrue(queryId != "") - - // Sleep 5 seconds to make sure profile collection is done - Thread.sleep(5000) - - def String profileContent = getProfile(queryId).toString() - logger.info("query {} profile\n{}", queryId, profileContent) - assertTrue(profileContent.contains("parallel_pipeline_task_num | 8")) -} \ No newline at end of file