Feature: Porting unique index for AO table to CBDB

gpcontrib/gp_internal_tools/gp_ao_co_diagnostics.c

@@ -48,6 +48,9 @@ gp_aoseg(PG_FUNCTION_ARGS);
 extern Datum
 gp_aocsseg_history(PG_FUNCTION_ARGS);
 
+extern Datum
+gp_aoblkdir(PG_FUNCTION_ARGS);
+
 extern Datum
 gp_aovisimap(PG_FUNCTION_ARGS);
 
@@ -67,6 +70,7 @@ PG_FUNCTION_INFO_V1(gp_aoseg_history_wrapper);
 PG_FUNCTION_INFO_V1(gp_aoseg_wrapper);
 PG_FUNCTION_INFO_V1(gp_aocsseg_wrapper);
 PG_FUNCTION_INFO_V1(gp_aocsseg_history_wrapper);
+PG_FUNCTION_INFO_V1(gp_aoblkdir_wrapper);
 PG_FUNCTION_INFO_V1(gp_aovisimap_wrapper);
 PG_FUNCTION_INFO_V1(gp_aovisimap_entry_wrapper);
 PG_FUNCTION_INFO_V1(gp_aovisimap_hidden_info_wrapper);
@@ -84,6 +88,8 @@ gp_aocsseg_wrapper(PG_FUNCTION_ARGS);
 extern Datum
 gp_aocsseg_history_wrapper(PG_FUNCTION_ARGS);
 extern Datum
+gp_aoblkdir_wrapper(PG_FUNCTION_ARGS);
+extern Datum
 gp_aovisimap_wrapper(PG_FUNCTION_ARGS);
 extern Datum
 gp_aovisimap_entry_wrapper(PG_FUNCTION_ARGS);
@@ -228,6 +234,21 @@ gp_aocsseg_history_wrapper(PG_FUNCTION_ARGS)
   PG_RETURN_DATUM(returnValue);
 }
 
+/*
+ * Interface to gp_aoblkdir_wrapper function.
+ *
+ * CREATE FUNCTION gp_aoblkdir_wrapper(regclass) RETURNS TABLE
+ * (segno integer, columngroup_no integer, first_row_no bigint, file_offset bigint, row_count bigint)
+ * AS '$libdir/gp_ao_co_diagnostics.so', 'gp_aoblkdir_wrapper' LANGUAGE C STRICT;
+ */
+Datum
+gp_aoblkdir_wrapper(PG_FUNCTION_ARGS)
+{
+	Datum returnValue = gp_aoblkdir(fcinfo);
+
+	PG_RETURN_DATUM(returnValue);
+}
+
 /* 
  * Interface to gp_aovisimap_wrapper function.
  *

src/backend/access/aocs/aocs_compaction.c

@@ -282,6 +282,14 @@ AOCSSegmentFileFullCompaction(Relation aorel,
 	estate->es_opened_result_relations =
 			lappend(estate->es_opened_result_relations, resultRelInfo);
 
+	/*
+	 * We don't want uniqueness checks to be performed while "insert"ing tuples
+	 * to a destination segfile during AOCSMoveTuple(). This is to ensure that
+	 * we can avoid spurious conflicts between the moved tuple and the original
+	 * tuple.
+	 */
+	estate->gp_bypass_unique_check = true;
+
 	while (aocs_getnext(scanDesc, ForwardScanDirection, slot))
 	{
 		CHECK_FOR_INTERRUPTS();

src/backend/access/aocs/aocsam.c

@@ -868,7 +868,7 @@ aocs_getnext(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 			/*
 			 * Perform any required upgrades on the Datum we just fetched.
 			 */
-			if (curseginfo->formatversion < AORelationVersion_GetLatest())
+			if (curseginfo->formatversion < AOSegfileFormatVersion_GetLatest ())
 			{
 				upgrade_datum_scan(scan, attno, d, null,
 								   curseginfo->formatversion);
@@ -1272,6 +1272,10 @@ positionSkipCurrentBlock(DatumStreamFetchDesc datumStreamFetchDesc)
 		datumStreamFetchDesc->currentBlock.lastRowNum + 1;
 }
 
+/*
+ * Fetch the tuple's datum from the block indicated by the block directory entry
+ * that covers the tuple, given the colno.
+ */
 static void
 fetchFromCurrentBlock(AOCSFetchDesc aocsFetchDesc,
 					  int64 rowNum,
@@ -1313,7 +1317,7 @@ fetchFromCurrentBlock(AOCSFetchDesc aocsFetchDesc,
 		/*
 		 * Perform any required upgrades on the Datum we just fetched.
 		 */
-		if (formatversion < AORelationVersion_GetLatest())
+		if (formatversion < AOSegfileFormatVersion_GetLatest ())
 		{
 			upgrade_datum_fetch(aocsFetchDesc, colno, values, nulls,
 								formatversion);
@@ -1331,14 +1335,49 @@ scanToFetchValue(AOCSFetchDesc aocsFetchDesc,
 				 TupleTableSlot *slot,
 				 int colno)
 {
-	DatumStreamFetchDesc datumStreamFetchDesc = aocsFetchDesc->datumStreamFetchDesc[colno];
-	DatumStreamRead *datumStream = datumStreamFetchDesc->datumStream;
-	bool		found;
+	DatumStreamFetchDesc 			datumStreamFetchDesc = aocsFetchDesc->datumStreamFetchDesc[colno];
+	DatumStreamRead 				*datumStream = datumStreamFetchDesc->datumStream;
+	CurrentBlock 			*currentBlock = &datumStreamFetchDesc->currentBlock;
+	AppendOnlyBlockDirectoryEntry 	*entry = &currentBlock->blockDirectoryEntry;
+	bool							found;
 
 	found = datumstreamread_find_block(datumStream,
 									   datumStreamFetchDesc,
 									   rowNum);
-	if (found)
+	if (!found)
+	{
+		if (AppendOnlyBlockDirectoryEntry_RangeHasRow(entry, rowNum))
+		{
+			/*
+			 * We fell into a hole inside the resolved block directory entry
+			 * we obtained from AppendOnlyBlockDirectory_GetEntry().
+			 * This should not be happening for versions >= CB2. Scream
+			 * appropriately. See AppendOnlyBlockDirectoryEntry for details.
+			 */
+			ereportif(AORelationVersion_Get(aocsFetchDesc->relation) >= AORelationVersion_CB2,
+					  ERROR,
+					  (errcode(ERRCODE_INTERNAL_ERROR),
+					   errmsg("datum with row number %ld and col no %d not found in block directory entry range", rowNum, colno),
+					   errdetail("block directory entry: (fileOffset = %ld, firstRowNum = %ld, "
+								 "afterFileOffset = %ld, lastRowNum = %ld)",
+								 entry->range.fileOffset,
+								 entry->range.firstRowNum,
+								 entry->range.afterFileOffset,
+								 entry->range.lastRowNum)));
+		}
+		else
+		{
+			/*
+			 * The resolved block directory entry we obtained from
+			 * AppendOnlyBlockDirectory_GetEntry() has range s.t.
+			 * firstRowNum < lastRowNum < rowNum
+			 * This can happen when rowNum maps to an aborted transaction, and
+			 * we find an earlier committed block directory row due to the
+			 * <= scan condition in AppendOnlyBlockDirectory_GetEntry().
+			 */
+		}
+	}
+	else
 		fetchFromCurrentBlock(aocsFetchDesc, rowNum, slot, colno);
 
 	return found;
@@ -1412,6 +1451,11 @@ openFetchSegmentFile(AOCSFetchDesc aocsFetchDesc,
 	return true;
 }
 
+/*
+ * Note: we don't reset the block directory entry here. This is crucial, so we
+ * can use the block directory entry later on. See comment in AOFetchBlockMetadata
+ * FIXME: reset other fields here.
+ */
 static void
 resetCurrentBlockInfo(CurrentBlock *currentBlock)
 {
@@ -1815,119 +1859,6 @@ aocs_fetch_finish(AOCSFetchDesc aocsFetchDesc)
 	AppendOnlyVisimap_Finish(&aocsFetchDesc->visibilityMap, AccessShareLock);
 }
 
-typedef struct AOCSUpdateDescData
-{
-	AOCSInsertDesc insertDesc;
-
-	/*
-	 * visibility map
-	 */
-	AppendOnlyVisimap visibilityMap;
-
-	/*
-	 * Visimap delete support structure. Used to handle out-of-order deletes
-	 */
-	AppendOnlyVisimapDelete visiMapDelete;
-
-}			AOCSUpdateDescData;
-
-AOCSUpdateDesc
-aocs_update_init(Relation rel, int segno)
-{
-	Oid			visimaprelid;
-	Oid			visimapidxid;
-	AOCSUpdateDesc desc = (AOCSUpdateDesc) palloc0(sizeof(AOCSUpdateDescData));
-
-	desc->insertDesc = aocs_insert_init(rel, segno);
-
-    GetAppendOnlyEntryAuxOids(rel->rd_id,
-                              desc->insertDesc->appendOnlyMetaDataSnapshot,
-                              NULL, NULL, NULL,
-                              &visimaprelid, &visimapidxid);
-	AppendOnlyVisimap_Init(&desc->visibilityMap,
-						   visimaprelid,
-						   visimapidxid,
-						   RowExclusiveLock,
-						   desc->insertDesc->appendOnlyMetaDataSnapshot);
-
-	AppendOnlyVisimapDelete_Init(&desc->visiMapDelete,
-								 &desc->visibilityMap);
-
-	return desc;
-}
-
-void
-aocs_update_finish(AOCSUpdateDesc desc)
-{
-	Assert(desc);
-
-	AppendOnlyVisimapDelete_Finish(&desc->visiMapDelete);
-
-	aocs_insert_finish(desc->insertDesc, NULL);
-	desc->insertDesc = NULL;
-
-	/* Keep lock until the end of transaction */
-	AppendOnlyVisimap_Finish(&desc->visibilityMap, NoLock);
-
-	pfree(desc);
-}
-
-TM_Result
-aocs_update(AOCSUpdateDesc desc, TupleTableSlot *slot,
-			AOTupleId *oldTupleId, AOTupleId *newTupleId)
-{
-	TM_Result result;
-
-	Assert(desc);
-	Assert(oldTupleId);
-	Assert(newTupleId);
-
-#ifdef FAULT_INJECTOR
-	FaultInjector_InjectFaultIfSet(
-								   "appendonly_update",
-								   DDLNotSpecified,
-								   "", //databaseName
-								   RelationGetRelationName(desc->insertDesc->aoi_rel));
-	/* tableName */
-#endif
-
-	result = AppendOnlyVisimapDelete_Hide(&desc->visiMapDelete, oldTupleId);
-	if (result != TM_Ok)
-		return result;
-
-	slot_getallattrs(slot);
-	aocs_insert_values(desc->insertDesc,
-					   slot->tts_values, slot->tts_isnull,
-					   newTupleId);
-
-	return result;
-}
-
-
-/*
- * AOCSDeleteDescData is used for delete data from AOCS relations.
- * It serves an equivalent purpose as AppendOnlyScanDescData
- * (relscan.h) only that the later is used for scanning append-only
- * relations.
- */
-typedef struct AOCSDeleteDescData
-{
-	/*
-	 * Relation to delete from
-	 */
-	Relation	aod_rel;
-
-	/*
-	 * visibility map
-	 */
-	AppendOnlyVisimap visibilityMap;
-
-	/*
-	 * Visimap delete support structure. Used to handle out-of-order deletes
-	 */
-	AppendOnlyVisimapDelete visiMapDelete;
-
-}			AOCSDeleteDescData;
 
 
 /*
@@ -2186,7 +2117,7 @@ aocs_addcol_newsegfile(AOCSAddColumnDesc desc,
 		int			version;
 
 		/* Always write in the latest format */
-		version = AORelationVersion_GetLatest();
+		version = AOSegfileFormatVersion_GetLatest();
 
 		FormatAOSegmentFileName(basepath, seginfo->segno, colno,
 								&fileSegNo, fn);
@@ -2674,7 +2605,7 @@ aocs_getnext_sample(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *
 			/*
 			 * Perform any required upgrades on the Datum we just fetched.
 			 */
-			if (curseginfo->formatversion < AORelationVersion_GetLatest())
+			if (curseginfo->formatversion < AOSegfileFormatVersion_GetLatest ())
 			{
 				upgrade_datum_scan(scan, attno, d, null,
 								   curseginfo->formatversion);