Open proper AO/AOCS segment files according to data volume

.github/workflows/build.yml

@@ -36,7 +36,7 @@ jobs:
     needs: build
     runs-on: [self-hosted, example]
     env:
-      MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=off -c gp_appendonly_insert_files=0' installcheck-world"
+      MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=off' installcheck-world"
       TEST_OS: "centos"
       DUMP_DB: "true"
     steps:
@@ -83,7 +83,7 @@ jobs:
     needs: build
     runs-on: [self-hosted, example]
     env:
-      MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=on -c gp_appendonly_insert_files=0' installcheck-world"
+      MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=on' installcheck-world"
       TEST_OS: "centos"
       DUMP_DB: "true"
     steps:
@@ -130,7 +130,7 @@ jobs:
     needs: build
     runs-on: [self-hosted, example]
     env:
-      MAKE_TEST_COMMAND: "-C src/test/regress -k PGOPTIONS='-c gp_appendonly_insert_files=0' installcheck-cbdb-parallel"
+      MAKE_TEST_COMMAND: "-C src/test/regress installcheck-cbdb-parallel"
       TEST_OS: "centos"
       DUMP_DB: "true"
     steps:

.github/workflows/release.yml

@@ -39,7 +39,7 @@ jobs:
     needs: build
     runs-on: [self-hosted, example]
     env:
-      MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=off -c gp_appendonly_insert_files=0' installcheck-world"
+      MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=off' installcheck-world"
       TEST_OS: "centos"
       DUMP_DB: "true"
       BUILD_TYPE: release
@@ -95,7 +95,7 @@ jobs:
     needs: build
     runs-on: [self-hosted, example]
     env:
-      MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=on -c gp_appendonly_insert_files=0' installcheck-world"
+      MAKE_TEST_COMMAND: "-k PGOPTIONS='-c optimizer=on' installcheck-world"
       TEST_OS: "centos"
       DUMP_DB: "true"
       BUILD_TYPE: release
@@ -151,7 +151,7 @@ jobs:
     needs: build
     runs-on: [self-hosted, example]
     env:
-      MAKE_TEST_COMMAND: "-C src/test/regress -k PGOPTIONS='-c gp_appendonly_insert_files=0' installcheck-cbdb-parallel"
+      MAKE_TEST_COMMAND: "-C src/test/regress installcheck-cbdb-parallel"
       TEST_OS: "centos"
       DUMP_DB: "true"
     steps:

src/backend/access/aocs/aocsam.c

@@ -986,7 +986,6 @@ aocs_insert_init(Relation rel, int segno)
 	desc = (AOCSInsertDesc) palloc0(sizeof(AOCSInsertDescData));
 	desc->aoi_rel = rel;
 	desc->appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));
-	desc->insertMultiFiles = false;
 
 	/*
 	 * Writers uses this since they have exclusive access to the lock acquired
@@ -1047,9 +1046,6 @@ aocs_insert_init(Relation rel, int segno)
 											(FileSegInfo *) desc->fsInfo, desc->lastSequence,
 											rel, segno, tupleDesc->natts, true);
 
-	/* Should not enable insertMultiFiles if the table is created by own transaction or in utility mode */
-	if (Gp_role != GP_ROLE_UTILITY)
-		desc->insertMultiFiles = gp_appendonly_insert_files > 1 && !ShouldUseReservedSegno(rel, CHOOSE_MODE_WRITE);
 	return desc;
 }
 

src/backend/access/aocs/aocsam_handler.c

@@ -88,9 +88,18 @@ typedef struct AOCODMLState
 {
 	Oid				relationOid;
 	AOCSInsertDesc	insertDesc;
-	dlist_head		head; // Head of multiple segment files insertion list.
 	AOCSDeleteDesc	deleteDesc;
 	AOCSUniqueCheckDesc uniqueCheckDesc;
+	/*
+	 * CBDB_PARALLEL
+	 * head: the Head of multiple segment files insertion list.
+	 * insertMultiFiles: number of seg files to be inserted into.
+	 * used_segment_files: used to avoid used files when asking
+	 * for a new segment file.
+	 */
+	dlist_head		head;
+	int 			insertMultiFiles;
+	List* 			used_segment_files;
 } AOCODMLState;
 
 static void reset_state_cb(void *arg);
@@ -191,6 +200,8 @@ enter_dml_state(const Oid relationOid)
 	state->insertDesc = NULL;
 	state->deleteDesc = NULL;
 	state->uniqueCheckDesc = NULL;
+	state->insertMultiFiles = 0;
+	state->used_segment_files = NIL;
 	dlist_init(&state->head);
 
 	Assert(!found);
@@ -341,48 +352,41 @@ get_insert_descriptor(const Relation relation)
 
 	if (state->insertDesc == NULL)
 	{
-		List	*segments = NIL;
 		MemoryContext oldcxt;
 
+		/*
+		 * CBDB_PARALLEL:
+		 * Should not enable insertMultiFiles if the table is created by own transaction
+		 * or in utility mode.
+		 */
+		if (Gp_role != GP_ROLE_UTILITY &&
+			gp_appendonly_insert_files > 1 &&
+			!ShouldUseReservedSegno(relation, CHOOSE_MODE_WRITE))
+			state->insertMultiFiles = gp_appendonly_insert_files;
+
 		oldcxt = MemoryContextSwitchTo(aocoLocal.stateCxt);
 		state->insertDesc = aocs_insert_init(relation,
 									  ChooseSegnoForWrite(relation));
-
-		dlist_init(&state->head);
-		dlist_head *head = &state->head;
-		dlist_push_tail(head, &state->insertDesc->node);
-
-		if (state->insertDesc->insertMultiFiles)
-		{
-			segments = lappend_int(segments, state->insertDesc->cur_segno);
-			for (int i = 0; i < gp_appendonly_insert_files - 1; i++)
-			{
-				next = aocs_insert_init(relation,
-												 ChooseSegnoForWriteMultiFile(relation, segments));
-				dlist_push_tail(head, &next->node);
-				segments = lappend_int(segments, next->cur_segno);
-			}
-			list_free(segments);
-		}
 
-		//* mark all insertDesc  placeholderInserted with false */
-        if (relationHasUniqueIndex(relation))
-        {
-            dlist_iter          iter;
-            dlist_foreach(iter, head)
-            {
-                AOCSInsertDesc insertDesc = (AOCSInsertDesc)dlist_container(AOCSInsertDescData, node, iter.cur);
-                insertDesc->placeholderInserted = false;
-            }
-        }
+		state->used_segment_files = list_make1_int(state->insertDesc->cur_segno);
+		dlist_init(&state->head);
+		dlist_push_tail(&state->head, &state->insertDesc->node);
 
 		MemoryContextSwitchTo(oldcxt);
 	}
 
 	/* switch insertDesc */
-	if (state->insertDesc->insertMultiFiles && state->insertDesc->range == gp_appendonly_insert_files_tuples_range)
+	if (state->insertMultiFiles && state->insertDesc->range == gp_appendonly_insert_files_tuples_range)
 	{
 		state->insertDesc->range = 0;
+
+		if (list_length(state->used_segment_files) < state->insertMultiFiles)
+		{
+			next = aocs_insert_init(relation, ChooseSegnoForWriteMultiFile(relation, state->used_segment_files));
+			dlist_push_tail(&state->head, &next->node);
+			state->used_segment_files = lappend_int(state->used_segment_files, next->cur_segno);
+		}
+
 		if (!dlist_has_next(&state->head, &state->insertDesc->node))
 			next = (AOCSInsertDesc)dlist_container(AOCSInsertDescData, node, dlist_head_node(&state->head));
 		else

src/backend/access/appendonly/appendonlyam.c

@@ -2634,7 +2634,6 @@ appendonly_insert_init(Relation rel, int segno)
 
 	aoInsertDesc->aoi_rel = rel;
 	aoInsertDesc->range = 0;
-	aoInsertDesc->insertMultiFiles = false;
 
 	/*
 	 * We want to see an up-to-date view of the metadata. The target segment's
@@ -2822,9 +2821,6 @@ aoInsertDesc->appendOnlyMetaDataSnapshot, //CONCERN:Safe to assume all block dir
 											aoInsertDesc->fsInfo, aoInsertDesc->lastSequence,
 											rel, segno, 1, false);
 
-	/* Should not enable insertMultiFiles if the table is created by own transaction or in utility mode */
-	if (Gp_role != GP_ROLE_UTILITY)
-		aoInsertDesc->insertMultiFiles = gp_appendonly_insert_files > 1 && !ShouldUseReservedSegno(rel, CHOOSE_MODE_WRITE);
 	return aoInsertDesc;
 }
 

src/backend/access/appendonly/appendonlyam_handler.c

@@ -59,9 +59,19 @@ typedef struct AppendOnlyDMLState
 {
 	Oid relationOid;
 	AppendOnlyInsertDesc	insertDesc;
-	dlist_head				head; // Head of multiple segment files insertion list.
 	AppendOnlyDeleteDesc	deleteDesc;
 	AppendOnlyUniqueCheckDesc uniqueCheckDesc;
+
+	/*
+	 * CBDB_PARALLEL
+	 * head: the Head of multiple segment files insertion list.
+	 * insertMultiFiles: number of seg files to be inserted into.
+	 * used_segment_files: used to avoid used files when asking
+	 * for a new segment file.
+	 */
+	dlist_head		head;
+	int 			insertMultiFiles;
+	List* 			used_segment_files;
 } AppendOnlyDMLState;
 
 
@@ -162,6 +172,8 @@ enter_dml_state(const Oid relationOid)
 	state->insertDesc = NULL;
 	state->deleteDesc = NULL;
 	state->uniqueCheckDesc = NULL;
+	state->insertMultiFiles = 0;
+	state->used_segment_files = NIL;
 	dlist_init(&state->head);
 
 	Assert(!found);
@@ -326,48 +338,41 @@ get_insert_descriptor(const Relation relation)
 
 	if (state->insertDesc == NULL)
 	{
-		List	*segments = NIL;
 		MemoryContext oldcxt;
 
+		/*
+		 * CBDB_PARALLEL:
+		 * Should not enable insertMultiFiles if the table is created by own transaction
+		 * or in utility mode.
+		 */
+		if (Gp_role != GP_ROLE_UTILITY &&
+			gp_appendonly_insert_files > 1 &&
+			!ShouldUseReservedSegno(relation, CHOOSE_MODE_WRITE))
+			state->insertMultiFiles = gp_appendonly_insert_files;
+
 		oldcxt = MemoryContextSwitchTo(appendOnlyLocal.stateCxt);
 		state->insertDesc= appendonly_insert_init(relation,
 											ChooseSegnoForWrite(relation));
 
+		state->used_segment_files = list_make1_int(state->insertDesc->cur_segno);
 		dlist_init(&state->head);
-		dlist_head *head = &state->head;
-		dlist_push_tail(head, &state->insertDesc->node);
-
-		if (state->insertDesc->insertMultiFiles)
-		{
-			segments = lappend_int(segments, state->insertDesc->cur_segno);
-			for (int i = 0; i < gp_appendonly_insert_files - 1; i++)
-			{
-				next = appendonly_insert_init(relation,
-												ChooseSegnoForWriteMultiFile(relation, segments));
-				dlist_push_tail(head, &next->node);
-				segments = lappend_int(segments, next->cur_segno);
-			}
-			list_free(segments);
-		}
-
-		//* mark all insertDesc  placeholderInserted with false */
-        if (relationHasUniqueIndex(relation))
-        {
-            dlist_iter          iter;
-            dlist_foreach(iter, head)
-            {
-                AppendOnlyInsertDesc insertDesc = (AppendOnlyInsertDesc)dlist_container(AppendOnlyInsertDescData, node, iter.cur);
-                insertDesc->placeholderInserted = false;
-            }
-        }
+		dlist_push_tail(&state->head, &state->insertDesc->node);
 
 		MemoryContextSwitchTo(oldcxt);
 	}
 
 	/* switch insertDesc */
-	if (state->insertDesc->insertMultiFiles && state->insertDesc->range == gp_appendonly_insert_files_tuples_range)
+	if (state->insertMultiFiles && state->insertDesc->range == gp_appendonly_insert_files_tuples_range)
 	{
 		state->insertDesc->range = 0;
+
+		if (list_length(state->used_segment_files) < state->insertMultiFiles)
+		{
+			next = appendonly_insert_init(relation, ChooseSegnoForWriteMultiFile(relation, state->used_segment_files));
+			dlist_push_tail(&state->head, &next->node);
+			state->used_segment_files = lappend_int(state->used_segment_files, next->cur_segno);
+		}
+
 		if (!dlist_has_next(&state->head, &state->insertDesc->node))
 			next = (AppendOnlyInsertDesc)dlist_container(AppendOnlyInsertDescData, node, dlist_head_node(&state->head));
 		else

src/backend/utils/misc/guc_gp.c

@@ -3166,7 +3166,7 @@ struct config_int ConfigureNamesInt_gp[] =
 			NULL
 		},
 		&gp_appendonly_insert_files,
-		4 /* CBDB_PARALLEL If default value is changed, set it in src/test/regress/GNUMakefile too, see details there */, 0, 127,
+		4, 0, 127,
 		NULL, NULL, NULL
 	},
 

src/include/cdb/cdbaocsam.h

@@ -74,7 +74,6 @@ typedef struct AOCSInsertDescData
 	/*
 	 * For multiple segment files insertion.
 	 */
-	bool			insertMultiFiles; /* insert into multi files */
 	dlist_node		node;	/* node of segfiles list */
 	int 			range;  /* inserted tuples of each range */
 	/* flag for insert placeholder in unique index   */

src/include/cdb/cdbappendonlyam.h

@@ -112,10 +112,6 @@ typedef struct AppendOnlyInsertDescData
 	/* The block directory for the appendonly relation. */
 	AppendOnlyBlockDirectory blockDirectory;
 
-	/*
-	 * For multiple segment files insertion.
-	 */
-	bool			insertMultiFiles; /* insert into multi files */
 	dlist_node		node;	/* node of segfiles list */
 	int 			range;  /* inserted tuples of each range */
 	/* flag for insert placeholder in unique index   */

src/test/isolation2/Makefile

@@ -63,11 +63,7 @@ clean distclean:
 install: all gpdiff.pl gpstringsubs.pl
 
 installcheck: install installcheck-parallel-retrieve-cursor
-	(\
-	gpconfig -c gp_appendonly_insert_files -v 0; \
-	gpstop -u; \
-	$(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --schedule=$(srcdir)/isolation2_schedule \
-	)
+	$(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --schedule=$(srcdir)/isolation2_schedule
 
 installcheck-resgroup: install
 	$(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_resgroup --dbname=isolation2resgrouptest --schedule=$(srcdir)/isolation2_resgroup_schedule
@@ -78,8 +74,6 @@ installcheck-parallel-retrieve-cursor: install
 # Add a new rule for running installcheck with parallel mode enabled.
 installcheck-cbdb-parallel: install
 	(\
-	gpconfig -c gp_appendonly_insert_files -v 0; \
-	gpstop -u; \
 	export PGOPTIONS='-c optimizer=off -c enable_parallel=true'; \
 	$(pg_isolation2_regress_installcheck) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --schedule=$(srcdir)/isolation2_schedule \
 	)

src/test/isolation2/expected/ao_blkdir.out

@@ -58,9 +58,13 @@ SELECT (gp_toolkit.__gp_aoblkdir('ao_blkdir_test')).* FROM gp_dist_random('gp_id
 
 TRUNCATE ao_blkdir_test;
 TRUNCATE
+set gp_appendonly_insert_files = 0;
+SET
 -- Insert enough rows to overflow the first block directory minipage by 2.
 INSERT INTO ao_blkdir_test SELECT i, 2 FROM generate_series(1, 292700) i;
 INSERT 292700
+reset gp_appendonly_insert_files;
+RESET
 -- There should be 2 block directory rows, one with 161 entries covering 292698
 -- rows and the other with 1 entry covering the 2 overflow rows.
 SELECT (gp_toolkit.__gp_aoblkdir('ao_blkdir_test')).* FROM gp_dist_random('gp_id') WHERE gp_segment_id = 0 ORDER BY 1,2,3,4,5;
@@ -380,8 +384,12 @@ SELECT (gp_toolkit.__gp_aoblkdir('aoco_blkdir_test')).* FROM gp_dist_random('gp_
 TRUNCATE aoco_blkdir_test;
 TRUNCATE
 -- Insert enough rows to overflow the first block directory minipage by 2.
+set gp_appendonly_insert_files = 0;
+SET
 INSERT INTO aoco_blkdir_test SELECT i, 2 FROM generate_series(1, 1317143) i;
 INSERT 1317143
+reset gp_appendonly_insert_files;
+RESET
 -- There should be 2 block directory rows, 2 for each column, one with 161
 -- entries covering 1317141 rows and the other with 1 entry covering the 2
 -- overflow rows.

src/test/isolation2/expected/ao_unique_index.out

@@ -302,6 +302,8 @@ SELECT gp_inject_fault('insert_new_entry_curr_minipage_full', 'suspend', '', '',
  Success:        
  Success:        
 (3 rows)
+1: set gp_appendonly_insert_files = 0;
+SET
 1&: INSERT INTO unique_index_ao_row SELECT generate_series(1, 661510);  <waiting ...>
 
 -- Wait until we have inserted (2048 * (161 * 2 + 1) + 3) = 661507 rows and we

src/test/isolation2/expected/aocs_unique_index.out

@@ -302,6 +302,8 @@ SELECT gp_inject_fault('insert_new_entry_curr_minipage_full', 'suspend', '', '',
  Success:        
  Success:        
 (3 rows)
+1: set gp_appendonly_insert_files = 0;
+SET
 1&: INSERT INTO unique_index_ao_column SELECT generate_series(1, 1321075);  <waiting ...>
 
 -- Wait until we have inserted (4090 * (161 * 2 + 1) + 3) = 1321073 rows and we