From 28a9f7b06bd8dcaccdbb90710087b784bc7d83a9 Mon Sep 17 00:00:00 2001 From: xy720 Date: Thu, 12 May 2022 21:34:53 +0800 Subject: [PATCH 1/2] save code --- .../doris/analysis/CreateRoutineLoadStmt.java | 4 +- .../load/routineload/RoutineLoadJob.java | 70 ++++++++----------- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java index c4c6f78593314e..45a0a7ff42dcb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java @@ -461,8 +461,8 @@ private void checkJobProperties() throws UserException { format = ""; // if it's not json, then it's mean csv and set empty } else if (format.equalsIgnoreCase("json")) { format = "json"; - jsonPaths = jobProperties.get(JSONPATHS); - jsonRoot = jobProperties.get(JSONROOT); + jsonPaths = jobProperties.getOrDefault(JSONPATHS, ""); + jsonRoot = jobProperties.getOrDefault(JSONROOT, ""); stripOuterArray = Boolean.valueOf(jobProperties.getOrDefault(STRIP_OUTER_ARRAY, "false")); numAsString = Boolean.valueOf(jobProperties.getOrDefault(NUM_AS_STRING, "false")); fuzzyParse = Boolean.valueOf(jobProperties.getOrDefault(FUZZY_PARSE, "false")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 2c62ef004e0a0f..8d16e79e1c033a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -299,42 +299,37 @@ protected void setOptional(CreateRoutineLoadStmt stmt) throws UserException { if (Strings.isNullOrEmpty(stmt.getFormat()) || stmt.getFormat().equals("csv")) { jobProperties.put(PROPS_FORMAT, "csv"); - jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false"); - jobProperties.put(PROPS_NUM_AS_STRING, "false"); - jobProperties.put(PROPS_JSONPATHS, ""); - jobProperties.put(PROPS_JSONROOT, ""); - jobProperties.put(PROPS_FUZZY_PARSE, "false"); } else if (stmt.getFormat().equals("json")) { jobProperties.put(PROPS_FORMAT, "json"); - if (!Strings.isNullOrEmpty(stmt.getJsonPaths())) { - jobProperties.put(PROPS_JSONPATHS, stmt.getJsonPaths()); - } else { - jobProperties.put(PROPS_JSONPATHS, ""); - } - if (!Strings.isNullOrEmpty(stmt.getJsonRoot())) { - jobProperties.put(PROPS_JSONROOT, stmt.getJsonRoot()); - } else { - jobProperties.put(PROPS_JSONROOT, ""); - } - if (stmt.isStripOuterArray()) { - jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "true"); - } else { - jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false"); - } - if (stmt.isNumAsString()) { - jobProperties.put(PROPS_NUM_AS_STRING, "true"); - } else { - jobProperties.put(PROPS_NUM_AS_STRING, "false"); - } - if (stmt.isFuzzyParse()) { - jobProperties.put(PROPS_FUZZY_PARSE, "true"); - } else { - jobProperties.put(PROPS_FUZZY_PARSE, "false"); - } - } else { throw new UserException("Invalid format type."); } + + if (!Strings.isNullOrEmpty(stmt.getJsonPaths())) { + jobProperties.put(PROPS_JSONPATHS, stmt.getJsonPaths()); + } else { + jobProperties.put(PROPS_JSONPATHS, ""); + } + if (!Strings.isNullOrEmpty(stmt.getJsonRoot())) { + jobProperties.put(PROPS_JSONROOT, stmt.getJsonRoot()); + } else { + jobProperties.put(PROPS_JSONROOT, ""); + } + if (stmt.isStripOuterArray()) { + jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "true"); + } else { + jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false"); + } + if (stmt.isNumAsString()) { + jobProperties.put(PROPS_NUM_AS_STRING, "true"); + } else { + jobProperties.put(PROPS_NUM_AS_STRING, "false"); + } + if (stmt.isFuzzyParse()) { + jobProperties.put(PROPS_FUZZY_PARSE, "true"); + } else { + jobProperties.put(PROPS_FUZZY_PARSE, "false"); + } } private void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { @@ -342,11 +337,7 @@ private void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { columnDescs = new ImportColumnDescs(); if (routineLoadDesc.getColumnsInfo() != null) { ImportColumnsStmt columnsStmt = routineLoadDesc.getColumnsInfo(); - if (columnsStmt.getColumns() != null || columnsStmt.getColumns().size() != 0) { - for (ImportColumnDesc columnDesc : columnsStmt.getColumns()) { - columnDescs.descs.add(columnDesc); - } - } + columnDescs.descs.addAll(columnsStmt.getColumns()); } if (routineLoadDesc.getPrecedingFilter() != null) { precedingFilter = routineLoadDesc.getPrecedingFilter().getExpr(); @@ -1316,7 +1307,7 @@ public List getShowInfo() { public List> getTasksShowInfo() { List> rows = Lists.newArrayList(); - routineLoadTaskInfoList.stream().forEach(entity -> { + routineLoadTaskInfoList.forEach(entity -> { try { entity.setTxnStatus(Catalog.getCurrentCatalog().getGlobalTransactionMgr().getDatabaseTransactionMgr(dbId).getTransactionState(entity.getTxnId()).getTransactionStatus()); rows.add(entity.getTaskShowInfo()); @@ -1481,10 +1472,7 @@ public boolean needRemove() { return false; } Preconditions.checkState(endTimestamp != -1, endTimestamp); - if ((System.currentTimeMillis() - endTimestamp) > Config.label_keep_max_second * 1000) { - return true; - } - return false; + return (System.currentTimeMillis() - endTimestamp) > Config.label_keep_max_second * 1000; } public boolean isFinal() { From 914789367e8d0e13de12780bd5eecede60142fd0 Mon Sep 17 00:00:00 2001 From: xy720 Date: Thu, 12 May 2022 21:46:45 +0800 Subject: [PATCH 2/2] save --- .../org/apache/doris/load/routineload/RoutineLoadJob.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 8d16e79e1c033a..4ee92ca02552c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -337,7 +337,9 @@ private void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { columnDescs = new ImportColumnDescs(); if (routineLoadDesc.getColumnsInfo() != null) { ImportColumnsStmt columnsStmt = routineLoadDesc.getColumnsInfo(); - columnDescs.descs.addAll(columnsStmt.getColumns()); + if (columnsStmt.getColumns() != null) { + columnDescs.descs.addAll(columnsStmt.getColumns()); + } } if (routineLoadDesc.getPrecedingFilter() != null) { precedingFilter = routineLoadDesc.getPrecedingFilter().getExpr();