From 2f0855df35669def7d9baffba6d41f9a76663910 Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Tue, 21 Jan 2025 15:25:58 +0800 Subject: [PATCH 1/2] [fix](external catalog) Persisting the External Catalog comment field (#46946) Since the comment field is not persisted, the comment will be lost after the FE is restarted after the comment is modified. This PR adds persistence to the comment. --- .../doris/datasource/ExternalCatalog.java | 4 ++- .../doris/datasource/ExternalCatalogTest.java | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 778c110df0bfac..76939d54e7a8d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -134,6 +134,9 @@ public abstract class ExternalCatalog protected Map> idToDb = Maps.newConcurrentMap(); @SerializedName(value = "lastUpdateTime") protected long lastUpdateTime; + @SerializedName(value = "comment") + private String comment; + // db name does not contains "default_cluster" protected Map dbNameToId = Maps.newConcurrentMap(); private boolean objectCreated = false; @@ -142,7 +145,6 @@ public abstract class ExternalCatalog protected TransactionManager transactionManager; private ExternalSchemaCache schemaCache; - private String comment; // A cached and being converted properties for external catalog. // generated from catalog properties. private byte[] propLock = new byte[0]; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java index 70e5e5f37af6cb..c1a3231dbf9b2b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java @@ -282,4 +282,37 @@ public void testSerialization() throws Exception { file.delete(); } } + + @Test + public void testSerializationWithComment() throws Exception { + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT); + metaContext.setThreadLocalInfo(); + + // 1. Write objects to file + File file = new File("./external_catalog_with_comment_test.dat"); + file.createNewFile(); + DataOutputStream dos = new DataOutputStream(Files.newOutputStream(file.toPath())); + + TestExternalCatalog ctl = (TestExternalCatalog) mgr.getCatalog("test1"); + String testComment = "This is a test comment for serialization"; + ctl.setComment(testComment); // Set a custom comment value + ctl.write(dos); + dos.flush(); + dos.close(); + + // 2. Read objects from file + DataInputStream dis = new DataInputStream(Files.newInputStream(file.toPath())); + + TestExternalCatalog ctl2 = (TestExternalCatalog) ExternalCatalog.read(dis); + Configuration conf = ctl2.getConfiguration(); + Assertions.assertNotNull(conf); + + // Verify the comment is properly serialized and deserialized + Assertions.assertEquals(testComment, ctl2.getComment()); + + // 3. delete files + dis.close(); + file.delete(); + } } From c0d762d72ee8d8e93418541421389a5269ef6463 Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Tue, 11 Feb 2025 14:57:28 +0800 Subject: [PATCH 2/2] fix ut --- .../java/org/apache/doris/datasource/ExternalCatalogTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java index c1a3231dbf9b2b..ebf69c6df52388 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java @@ -304,7 +304,8 @@ public void testSerializationWithComment() throws Exception { // 2. Read objects from file DataInputStream dis = new DataInputStream(Files.newInputStream(file.toPath())); - TestExternalCatalog ctl2 = (TestExternalCatalog) ExternalCatalog.read(dis); + String json = Text.readString(dis); + TestExternalCatalog ctl2 = GsonUtils.GSON.fromJson(json, TestExternalCatalog.class); Configuration conf = ctl2.getConfiguration(); Assertions.assertNotNull(conf);