From 8490fee21c6f3e2a6c73604cc0bde43ed1c37051 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 31 Jul 2024 16:32:25 +0800 Subject: [PATCH 01/23] fix batch retry --- .../model/partition/ObRangePartDesc.java | 11 +- .../rpc/table/ObTableClientBatchOpsImpl.java | 182 ++++++++++++------ .../table/ObTableClientLSBatchOpsImpl.java | 84 ++++---- .../rpc/table/ObTableClientQueryImpl.java | 4 +- 4 files changed, 176 insertions(+), 105 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDesc.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDesc.java index b41871e9..e17d7bf2 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDesc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDesc.java @@ -66,7 +66,8 @@ public ObRangePartDesc() { public List getOrderedCompareColumnTypes() { return orderedCompareColumnTypes; } - private List completeWorks; + + private List completeWorks; /* * Set ordered compare column types. @@ -299,12 +300,12 @@ public int getBoundsIdx(boolean isScan, Row rowKey) { try { List evalParams = evalRowKeyValues(rowKey); List comparableElement = super.initComparableElementByTypes(evalParams, - this.orderedCompareColumns); + this.orderedCompareColumns); ObPartitionKey searchKey = ObPartitionKey.getInstance(orderedCompareColumns, - comparableElement); + comparableElement); int pos = upperBound(this.bounds, new ObComparableKV(searchKey, - (long) -1)); + (long) -1)); if (pos >= this.bounds.size()) { if (isScan) { // if range is bigger than rangeMax while scanning @@ -312,7 +313,7 @@ public int getBoundsIdx(boolean isScan, Row rowKey) { return this.bounds.size() - 1; } throw new ArrayIndexOutOfBoundsException("Table has no partition for value in " - + this.getPartExpr()); + + this.getPartExpr()); } else { return pos; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java index 857c3d5e..f1c476a5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java @@ -217,25 +217,41 @@ public List executeWithResult() throws Exception { return results; } - public Map>>> partitionPrepare() - throws Exception { - // consistent can not be sure - List operations = batchOperation.getTableOperations(); - Map>>> partitionOperationsMap = new HashMap>>>(); + // Helper method to calculate RowKey from ObTableOperation + private Object[] calculateRowKey(ObTableOperation operation) { + ObRowKey rowKeyObject = operation.getEntity().getRowKey(); + int rowKeySize = rowKeyObject.getObjs().size(); + Object[] rowKey = new Object[rowKeySize]; + for (int j = 0; j < rowKeySize; j++) { + rowKey[j] = rowKeyObject.getObj(j).getValue(); + } + return rowKey; + } + public List extractOperations(List> operationsPairs) { + List operations = new ArrayList<>(operationsPairs.size()); + for (ObPair pair : operationsPairs) { + operations.add(pair.getRight()); + } + return operations; + } + + public Map>>> prepareOperations(List operations) throws Exception { + Map>>> partitionOperationsMap = new HashMap<>(); + if (obTableClient.isOdpMode()) { - ObPair>> obTableOperations = new ObPair>>( - new ObTableParam(obTableClient.getOdpTable()), - new ArrayList>()); + ObPair>> obTableOperations = new ObPair<>( + new ObTableParam(obTableClient.getOdpTable()), + new ArrayList<>()); for (int i = 0; i < operations.size(); i++) { ObTableOperation operation = operations.get(i); obTableOperations.getRight().add( - new ObPair(i, operation)); + new ObPair<>(i, operation)); } partitionOperationsMap.put(0L, obTableOperations); return partitionOperationsMap; } - + for (int i = 0; i < operations.size(); i++) { ObTableOperation operation = operations.get(i); ObRowKey rowKeyObject = operation.getEntity().getRowKey(); @@ -248,18 +264,20 @@ public Map>>> tableName, rowKey, false, false, obTableClient.getRoute(batchOperation.isReadOnly())); ObPair>> obTableOperations = partitionOperationsMap - .get(tableObPair.getLeft()); - if (obTableOperations == null) { - obTableOperations = new ObPair>>( - tableObPair.getRight(), new ArrayList>()); - partitionOperationsMap.put(tableObPair.getLeft(), obTableOperations); - } - obTableOperations.getRight().add(new ObPair(i, operation)); + .computeIfAbsent(tableObPair.getLeft(), k -> new ObPair<>( + tableObPair.getRight(), new ArrayList<>())); + obTableOperations.getRight().add(new ObPair<>(i, operation)); } - return partitionOperationsMap; } + public Map>>> partitionPrepare() + throws Exception { + // consistent can not be sure + List operations = batchOperation.getTableOperations(); + return prepareOperations(operations); + } + /* * Partition execute. */ @@ -363,6 +381,7 @@ public void partitionExecute(ObTableOperationResult[] results, } catch (Exception ex) { if (obTableClient.isOdpMode()) { if ((tryTimes - 1) < obTableClient.getRuntimeRetryTimes()) { + assert ex instanceof ObTableException; logger .warn( "batch ops execute while meet Exception, tablename:{}, errorCode: {} , errorMsg: {}, try times {}", @@ -399,9 +418,9 @@ public void partitionExecute(ObTableOperationResult[] results, tableName, partId, ((ObTableException) ex).getErrorCode(), tryTimes, ex); if (ex instanceof ObTableNeedFetchAllException) { - needFetchAllRouteInfo = true; - // reset failure count while fetch all route info - obTableClient.resetExecuteContinuousFailureCount(tableName); + obTableClient.getOrRefreshTableEntry(tableName, needRefreshTableEntry, + obTableClient.isTableEntryRefreshIntervalWait(), true); + throw ex; } } else { obTableClient.calculateContinuousFailure(tableName, ex.getMessage()); @@ -483,50 +502,96 @@ public void partitionExecute(ObTableOperationResult[] results, obTableClient.getslowQueryMonitorThreshold()); } - /* - * Execute internal. - */ - public ObTableBatchOperationResult executeInternal() throws Exception { + private boolean shouldRetry(Throwable throwable) { + return throwable instanceof ObTableNeedFetchAllException; + } + private void executeWithRetries(ObTableOperationResult[] results, Map.Entry>>> entry, int maxRetries) throws Exception { + int retryCount = 0; + boolean success = false; + + // 初始分区 + Map>>> currentPartitions = new HashMap<>(); + currentPartitions.put(entry.getKey(), entry.getValue()); + + while (retryCount < maxRetries && !success) { + boolean allPartitionsSuccess = true; + + for (Map.Entry>>> currentEntry : currentPartitions.entrySet()) { + try { + partitionExecute(results, currentEntry); + } catch (Exception e) { + if (shouldRetry(e)) { + retryCount++; + List failedOperations = extractOperations(currentEntry.getValue().getRight()); + currentPartitions = prepareOperations(failedOperations); + System.out.println("refresh partitions: currentPartitions: " + currentPartitions); + allPartitionsSuccess = false; + break; + } else { + throw e; + } + } + } + + if (allPartitionsSuccess) { + success = true; + } + } + + if (!success) { + throw new ObTableUnexpectedException("Failed to execute operation after retrying " + maxRetries + " times."); + } + } + + public ObTableBatchOperationResult executeInternal() throws Exception { if (tableName == null || tableName.isEmpty()) { throw new IllegalArgumentException("table name is null"); } long start = System.currentTimeMillis(); List operations = batchOperation.getTableOperations(); - ObTableOperationResult[] obTableOperationResults = null; - if (returnOneResult) { - obTableOperationResults = new ObTableOperationResult[1]; - } else { - obTableOperationResults = new ObTableOperationResult[operations.size()]; - } + ObTableOperationResult[] obTableOperationResults = returnOneResult ? new ObTableOperationResult[1] + : new ObTableOperationResult[operations.size()]; Map>>> partitions = partitionPrepare(); long getTableTime = System.currentTimeMillis(); final Map context = ThreadLocalMap.getContextMap(); + final int maxRetries = obTableClient.getRuntimeRetryTimes(); + + ConcurrentTaskExecutor executor = null; if (executorService != null && !executorService.isShutdown() && partitions.size() > 1) { - final ConcurrentTaskExecutor executor = new ConcurrentTaskExecutor(executorService, - partitions.size()); - for (final Map.Entry>>> entry : partitions - .entrySet()) { - ObTableOperationResult[] finalObTableOperationResults = obTableOperationResults; - executor.execute(new ConcurrentTask() { - /* - * Do task. - */ - @Override - public void doTask() { - try { - ThreadLocalMap.transmitContextMap(context); - partitionExecute(finalObTableOperationResults, entry); - } catch (Exception e) { - logger.error(LCD.convert("01-00026"), e); - executor.collectExceptions(e); - } finally { - ThreadLocalMap.reset(); + executor = new ConcurrentTaskExecutor(executorService, partitions.size()); + } + for (final Map.Entry>>> entry : partitions + .entrySet()) { + try { + if (executor != null) { + // Concurrent execution + ConcurrentTaskExecutor finalExecutor = executor; + executor.execute(new ConcurrentTask() { + @Override + public void doTask() { + try { + ThreadLocalMap.transmitContextMap(context); + executeWithRetries(obTableOperationResults, entry, maxRetries); + } catch (Exception e) { + logger.error(LCD.convert("01-00026"), e); + finalExecutor.collectExceptions(e); + } finally { + ThreadLocalMap.reset(); + } } - } - }); + }); + } else { + // Sequential execution + executeWithRetries(obTableOperationResults, entry, maxRetries); + } + } catch (Exception e) { + logger.error("Error executing retry: {}", entry.getKey(), e); + throw e; } + } + if (executor != null) { long estimate = obTableClient.getRuntimeBatchMaxWait() * 1000L * 1000L; try { while (estimate > 0) { @@ -538,7 +603,7 @@ public void doTask() { "Batch Concurrent Execute interrupted", e); } - if (executor.getThrowableList().size() > 0) { + if (!executor.getThrowableList().isEmpty()) { throw new ObTableUnexpectedException("Batch Concurrent Execute Error", executor.getThrowableList().get(0)); } @@ -553,22 +618,11 @@ public void doTask() { executor.stop(); } - if (executor.getThrowableList().size() > 0) { - throw new ObTableUnexpectedException("Batch Concurrent Execute Error", executor - .getThrowableList().get(0)); - } - if (!executor.isComplete()) { throw new ObTableUnexpectedException("Batch Concurrent Execute Error [" + obTableClient.getRpcExecuteTimeout() + "]/ms"); } - - } else { - for (final Map.Entry>>> entry : partitions - .entrySet()) { - partitionExecute(obTableOperationResults, entry); - } } ObTableBatchOperationResult batchOperationResult = new ObTableBatchOperationResult(); diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index d58c4f4e..6268972c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -311,18 +311,32 @@ public List executeWithResult() throws Exception { return results; } - public Map>>>> partitionPrepare() - throws Exception { - List operations = getSingleOperations(); - // map: >>> - Map>>>> lsOperationsMap = - new HashMap(); + private Object[] calculateRowKey(ObTableSingleOp operation) { + List rowKeyObject = operation.getRowkeyObjs(); + int rowKeySize = rowKeyObject.size(); + Object[] rowKey = new Object[rowKeySize]; + for (int j = 0; j < rowKeySize; j++) { + rowKey[j] = rowKeyObject.get(j).getValue(); + } + return rowKey; + } + + + private List extractOperations(List> operationsPairs) throws Exception { + List operations = new ArrayList<>(operationsPairs.size()); + for (ObPair pair : operationsPairs) { + operations.add(pair.getRight()); + } + return operations; + } + + public Map>>>> prepareOperations(List operations) throws Exception { + Map>>>> lsOperationsMap = new HashMap<>(); - // In ODP mode, client send the request to ODP directly without route if (obTableClient.isOdpMode()) { Map>>> tabletOperationsMap = new HashMap<>(); ObPair>> obTableOperations = - new ObPair(new ObTableParam(obTableClient.getOdpTable()), + new ObPair<>(new ObTableParam(obTableClient.getOdpTable()), new ArrayList>()); for (int i = 0; i < operations.size(); i++) { ObTableSingleOp operation = operations.get(i); @@ -335,12 +349,7 @@ public Map rowkeyObjs = operation.getRowkeyObjs(); - int rowKeySize = rowkeyObjs.size(); - Object[] rowKey = new Object[rowKeySize]; - for (int j = 0; j < rowKeySize; j++) { - rowKey[j] = rowkeyObjs.get(j).getValue(); - } + Object[] rowKey = calculateRowKey(operation); String real_tableName = tableName; if (this.entityType == ObTableEntityType.HKV && obTableClient.isTableGroupName(tableName)) { @@ -350,28 +359,26 @@ public Map>>> tabletOperations - = lsOperationsMap.get(lsId); - // if ls id not exists - if (tabletOperations == null) { - tabletOperations = new HashMap<>(); - lsOperationsMap.put(lsId, tabletOperations); - } + Map>>> tabletOperations + = lsOperationsMap.computeIfAbsent(lsId, k -> new HashMap<>()); + // if ls id not exists - ObPair>> singleOperations = - tabletOperations.get(tableObPair.getLeft()); - // if tablet id not exists - if (singleOperations == null) { - singleOperations = new ObPair<>(tableObPair.getRight(), new ArrayList<>()); - tabletOperations.put(tableObPair.getLeft(), singleOperations); - } + ObPair>> singleOperations = + tabletOperations.computeIfAbsent(tableObPair.getLeft(), k -> new ObPair<>(tableObPair.getRight(), new ArrayList<>())); + // if tablet id not exists - singleOperations.getRight().add(new ObPair(i, operation)); - } + singleOperations.getRight().add(new ObPair<>(i, operation)); + } - return lsOperationsMap; + return lsOperationsMap; } + public Map>>>> partitionPrepare() throws Exception { + List operations = getSingleOperations(); + return prepareOperations(operations); + } + + /* * Partition execute. */ @@ -380,7 +387,7 @@ public void partitionExecute(ObTableSingleOpResult[] results, throws Exception { long lsId = lsOperation.getKey(); Map>>> tabletOperationsMap = lsOperation.getValue(); - if (tabletOperationsMap.size() == 0) { + if (tabletOperationsMap.isEmpty()) { logger.warn("the size of tablet operations in ls operation is zero"); throw new ObTableUnexpectedException("the size of tablet operations in ls operation is zero"); } @@ -515,6 +522,11 @@ public void partitionExecute(ObTableSingleOpResult[] results, logger.warn("tablename:{} ls id:{} batch ops retry while meet ObTableMasterChangeException, errorCode: {} , retry times {}", tableName, lsId, ((ObTableException) ex).getErrorCode(), tryTimes, ex); + if (ex instanceof ObTableNeedFetchAllException) { + obTableClient.getOrRefreshTableEntry(tableName, needRefreshTableEntry, + obTableClient.isTableEntryRefreshIntervalWait(), true); + throw ex; + } } else { obTableClient.calculateContinuousFailure(tableName, ex.getMessage()); throw ex; @@ -600,11 +612,15 @@ public void partitionExecute(ObTableSingleOpResult[] results, obTableClient.getslowQueryMonitorThreshold()); } + private boolean shouldRetry(Throwable throwable) { + return throwable instanceof ObTableNeedFetchAllException; + } + /* * Execute internal. */ public ObTableSingleOpResult[] executeInternal() throws Exception { - + final int maxRetries = obTableClient.getRuntimeRetryTimes(); if (tableName == null || tableName.isEmpty()) { throw new IllegalArgumentException("table name is null"); } @@ -633,7 +649,7 @@ public ObTableSingleOpResult[] executeInternal() throws Exception { public void doTask() { try { ThreadLocalMap.transmitContextMap(context); - partitionExecute(finalObTableOperationResults, entry); + executeWithRetries(finalObTableOperationResults, entry, maxRetries); } catch (Exception e) { logger.error(LCD.convert("01-00026"), e); executor.collectExceptions(e); diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java index ecf435d0..145fbbbf 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java @@ -274,8 +274,8 @@ public void initPartitions() throws Exception { ObBorderFlag borderFlag = rang.getBorderFlag(); // pairs -> List> List> pairs = this.obTableClient.getTables(indexTableName, - tableQuery, start, borderFlag.isInclusiveStart(), end, borderFlag.isInclusiveEnd(), - false, false); + tableQuery, start, borderFlag.isInclusiveStart(), end, borderFlag.isInclusiveEnd(), + false, false); if (this.tableQuery.getScanOrder() == ObScanOrder.Reverse) { for (int i = pairs.size() - 1; i >= 0; i--) { this.partitionObTables.put(pairs.get(i).getLeft(), pairs.get(i)); From 8bb956013c6c41b08211e6e3e5c32c8e325ea10c Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 5 Aug 2024 14:18:02 +0800 Subject: [PATCH 02/23] draft LSBatchImpl --- .../rpc/table/ObTableClientLSBatchOpsImpl.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index 6268972c..a288e787 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -321,7 +321,6 @@ private Object[] calculateRowKey(ObTableSingleOp operation) { return rowKey; } - private List extractOperations(List> operationsPairs) throws Exception { List operations = new ArrayList<>(operationsPairs.size()); for (ObPair pair : operationsPairs) { @@ -329,8 +328,8 @@ private List extractOperations(List>>>> prepareOperations(List operations) throws Exception { + + public Map>>>> prepareOperations(List operations) throws Exception { Map>>>> lsOperationsMap = new HashMap<>(); if (obTableClient.isOdpMode()) { @@ -373,11 +372,11 @@ public Map>>>> partitionPrepare() throws Exception { + public Map>>>> partitionPrepare() + throws Exception { List operations = getSingleOperations(); return prepareOperations(operations); } - /* * Partition execute. @@ -615,7 +614,7 @@ public void partitionExecute(ObTableSingleOpResult[] results, private boolean shouldRetry(Throwable throwable) { return throwable instanceof ObTableNeedFetchAllException; } - + /* * Execute internal. */ @@ -649,7 +648,7 @@ public ObTableSingleOpResult[] executeInternal() throws Exception { public void doTask() { try { ThreadLocalMap.transmitContextMap(context); - executeWithRetries(finalObTableOperationResults, entry, maxRetries); + partitionExecute(finalObTableOperationResults, entry); } catch (Exception e) { logger.error(LCD.convert("01-00026"), e); executor.collectExceptions(e); From 7765ad12328f96734a4327a2f8b08f92bafce7f2 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 5 Aug 2024 21:22:42 +0800 Subject: [PATCH 03/23] retry LSBatchImpl --- .../table/ObTableClientLSBatchOpsImpl.java | 64 ++++++++++++++++--- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index a288e787..08c2cf56 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -279,7 +279,7 @@ public void addOperation(Mutation mutation) throws Exception { * Execute. */ public List execute() throws Exception { - List results = new ArrayList(batchOperation.size()); + List results = new ArrayList<>(batchOperation.size()); for (ObTableSingleOpResult result : executeInternal()) { int errCode = result.getHeader().getErrno(); if (errCode == ResultCodes.OB_SUCCESS.errorCode) { @@ -321,10 +321,12 @@ private Object[] calculateRowKey(ObTableSingleOp operation) { return rowKey; } - private List extractOperations(List> operationsPairs) throws Exception { - List operations = new ArrayList<>(operationsPairs.size()); - for (ObPair pair : operationsPairs) { - operations.add(pair.getRight()); + private List extractOperations(Map>>> tabletOperationsMap) { + List operations = new ArrayList<>(); + for (ObPair>> pair : tabletOperationsMap.values()) { + for (ObPair operationWithIndex : pair.getRight()) { + operations.add(operationWithIndex.getRight()); + } } return operations; } @@ -541,7 +543,7 @@ public void partitionExecute(ObTableSingleOpResult[] results, long endExecute = System.currentTimeMillis(); if (subLSOpResult == null) { - RUNTIME.error("tablename:{} ls id:{} check batch operation result error: client get unexpected NULL result", + RUNTIME.error("table name:{} ls id:{} check batch operation result error: client get unexpected NULL result", tableName, lsId); throw new ObTableUnexpectedException("check batch operation result error: client get unexpected NULL result"); } @@ -615,11 +617,51 @@ private boolean shouldRetry(Throwable throwable) { return throwable instanceof ObTableNeedFetchAllException; } + private void executeWithRetries( + ObTableSingleOpResult[] results, + Map.Entry>>>> entry, + int maxRetries) throws Exception { + + int retryCount = 0; + boolean success = false; + + // 初始分区 + Map>>>> currentPartitions = new HashMap<>(); + currentPartitions.put(entry.getKey(), entry.getValue()); + + while (retryCount < maxRetries && !success) { + boolean allPartitionsSuccess = true; + + for (Map.Entry>>>> currentEntry : currentPartitions.entrySet()) { + try { + partitionExecute(results, currentEntry); + } catch (Exception e) { + if (shouldRetry(e)) { + retryCount++; + List failedOperations = extractOperations(currentEntry.getValue()); + currentPartitions = prepareOperations(failedOperations); + allPartitionsSuccess = false; + break; + } else { + throw e; + } + } + } + + if (allPartitionsSuccess) { + success = true; + } + } + + if (!success) { + throw new ObTableUnexpectedException("Failed to execute operation after retrying " + maxRetries + " times."); + } + } + /* * Execute internal. */ public ObTableSingleOpResult[] executeInternal() throws Exception { - final int maxRetries = obTableClient.getRuntimeRetryTimes(); if (tableName == null || tableName.isEmpty()) { throw new IllegalArgumentException("table name is null"); } @@ -633,6 +675,8 @@ public ObTableSingleOpResult[] executeInternal() throws Exception { Map>>>> lsOperations = partitionPrepare(); long getTableTime = System.currentTimeMillis(); final Map context = ThreadLocalMap.getContextMap(); + final int maxRetries = obTableClient.getRuntimeRetryTimes(); + if (executorService != null && !executorService.isShutdown() && lsOperations.size() > 1) { // execute sub-batch operation in parallel final ConcurrentTaskExecutor executor = new ConcurrentTaskExecutor(executorService, @@ -648,7 +692,7 @@ public ObTableSingleOpResult[] executeInternal() throws Exception { public void doTask() { try { ThreadLocalMap.transmitContextMap(context); - partitionExecute(finalObTableOperationResults, entry); + executeWithRetries(finalObTableOperationResults, entry, maxRetries); } catch (Exception e) { logger.error(LCD.convert("01-00026"), e); executor.collectExceptions(e); @@ -669,7 +713,7 @@ public void doTask() { "Batch Concurrent Execute interrupted", e); } - if (executor.getThrowableList().size() > 0) { + if (!executor.getThrowableList().isEmpty()) { throw new ObTableUnexpectedException("Batch Concurrent Execute Error", executor.getThrowableList().get(0)); } @@ -682,7 +726,7 @@ public void doTask() { executor.stop(); } - if (executor.getThrowableList().size() > 0) { + if (!executor.getThrowableList().isEmpty()) { throw new ObTableUnexpectedException("Batch Concurrent Execute Error", executor .getThrowableList().get(0)); } From 2eb9a4cd34944f4cad50614c21dd7e026c316414 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 7 Aug 2024 19:47:23 +0800 Subject: [PATCH 04/23] add error code for rerouting-exception --- .../oceanbase/rpc/bolt/transport/ObTableRemoting.java | 10 +++++----- .../oceanbase/rpc/table/ObTableClientBatchOpsImpl.java | 3 +-- .../rpc/table/ObTableClientLSBatchOpsImpl.java | 3 +-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java index e3d26f0d..4da53fb5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java @@ -127,16 +127,16 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques "routed to the wrong server: " + response.getMessage()); logger.warn(errMessage); if (needFetchAll(resultCode.getRcode(), resultCode.getPcode())) { - throw new ObTableNeedFetchAllException(errMessage); + throw new ObTableNeedFetchAllException(errMessage, resultCode.getRcode()); } else if (needFetchPartial(resultCode.getRcode())) { - throw new ObTableRoutingWrongException(errMessage); + throw new ObTableRoutingWrongException(errMessage, resultCode.getRcode()); } else { // Encountered an unexpected RoutingWrong error code, // possibly due to the client error code version being behind the observer's version. // Attempting a full refresh here // and delegating to the upper-level call to determine whether to throw the exception to the user based on the retry result. logger.warn("get unexpected error code: {}", response.getMessage()); - throw new ObTableNeedFetchAllException(errMessage); + throw new ObTableNeedFetchAllException(errMessage, resultCode.getRcode()); } } if (resultCode.getRcode() != 0 && response.getHeader().getPcode() != Pcodes.OB_TABLE_API_MOVE) { @@ -144,9 +144,9 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques "routed to the wrong server: " + response.getMessage()); logger.warn(errMessage); if (needFetchAll(resultCode.getRcode(), resultCode.getPcode())) { - throw new ObTableNeedFetchAllException(errMessage); + throw new ObTableNeedFetchAllException(errMessage, resultCode.getRcode()); } else if (needFetchPartial(resultCode.getRcode())) { - throw new ObTableRoutingWrongException(errMessage); + throw new ObTableRoutingWrongException(errMessage, resultCode.getRcode()); } else { ExceptionUtil.throwObTableException(conn.getObTable().getIp(), conn .getObTable().getPort(), response.getHeader().getTraceId1(), response diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java index f1c476a5..9e84ce2c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java @@ -509,8 +509,7 @@ private boolean shouldRetry(Throwable throwable) { private void executeWithRetries(ObTableOperationResult[] results, Map.Entry>>> entry, int maxRetries) throws Exception { int retryCount = 0; boolean success = false; - - // 初始分区 + Map>>> currentPartitions = new HashMap<>(); currentPartitions.put(entry.getKey(), entry.getValue()); diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index 08c2cf56..178f3fb3 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -624,8 +624,7 @@ private void executeWithRetries( int retryCount = 0; boolean success = false; - - // 初始分区 + Map>>>> currentPartitions = new HashMap<>(); currentPartitions.put(entry.getKey(), entry.getValue()); From de8f71f8be22df3469cc2eff8d8f3ccc5a76bad5 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 7 Aug 2024 19:54:18 +0800 Subject: [PATCH 05/23] remove unused debug log --- .../alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java index 9e84ce2c..5334eece 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java @@ -524,7 +524,6 @@ private void executeWithRetries(ObTableOperationResult[] results, Map.Entry failedOperations = extractOperations(currentEntry.getValue().getRight()); currentPartitions = prepareOperations(failedOperations); - System.out.println("refresh partitions: currentPartitions: " + currentPartitions); allPartitionsSuccess = false; break; } else { From e2649e0554093eeac7ae31aac5f6c9d229405cc0 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 12 Aug 2024 01:53:46 +0800 Subject: [PATCH 06/23] query refresh partitionTables --- .../query/AbstractQueryStreamResult.java | 96 ++++++++++++++++++- .../impl/execute/query/ObTableQuery.java | 82 ++++++++++++++++ .../ObTableClientQueryAsyncStreamResult.java | 24 ++++- .../ObTableClientQueryStreamResult.java | 7 ++ .../rpc/table/ObTableClientQueryImpl.java | 58 ++++++----- 5 files changed, 235 insertions(+), 32 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java index 7cd75788..45887026 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java @@ -29,6 +29,7 @@ import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableApiMove; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableStreamRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.QueryStreamResult; @@ -50,6 +51,7 @@ public abstract class AbstractQueryStreamResult extends AbstractPayload implemen protected volatile boolean closed = false; protected volatile List row = null; protected volatile int rowIndex = -1; + // 调整它的startKey protected ObTableQuery tableQuery; protected long operationTimeout = -1; protected String tableName; @@ -63,6 +65,7 @@ public abstract class AbstractQueryStreamResult extends AbstractPayload implemen protected LinkedList> cacheRows = new LinkedList>(); private LinkedList, ObTableQueryResult>> partitionLastResult = new LinkedList, ObTableQueryResult>>(); private ObReadConsistency readConsistency = ObReadConsistency.STRONG; + public ObRowKey currentStartKey; /* * Get pcode. @@ -243,6 +246,11 @@ protected ObPayload commonExecute(ObTableClient client, Logger logger, "tablename:{} partition id:{} stream query retry while meet Exception needing refresh, errorCode: {} , retry times {}", indexTableName, partIdWithIndex.getLeft(), ((ObTableException) e).getErrorCode(), tryTimes, e); + // tablet not exists, refresh table entry + if (e instanceof ObTableNeedFetchAllException) { + client.getOrRefreshTableEntry(tableName, true, true, true); + throw e; + } } else { client.calculateContinuousFailure(indexTableName, e.getMessage()); throw e; @@ -265,12 +273,13 @@ protected ObPayload commonExecute(ObTableClient client, Logger logger, /* * Next. */ + // 这个函数中，任何的计算结果都会填充到cacheRows public boolean next() throws Exception { checkStatus(); lock.lock(); try { // firstly, refer to the cache - if (cacheRows.size() > 0) { + if (!cacheRows.isEmpty()) { nextRow(); return true; } @@ -293,6 +302,40 @@ public boolean next() throws Exception { // lastly, refer to the new partition boolean hasNext = false; List>> referPartition = new ArrayList>>(); + Iterator>> it = expectant.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry> entry = it.next(); + referPartition.add(entry); + try { + // Mark the refer partition + referPartition.add(entry); + + // Try accessing the new partition + ObTableQueryResult tableQueryResult = (ObTableQueryResult) referToNewPartition(entry.getValue()); + + if (tableQueryResult.getRowCount() == 0) { + continue; + } + + hasNext = true; + nextRow(); + break; + + } catch (Exception e) { + if (e instanceof ObTableNeedFetchAllException) { + // Adjust the start key and refresh the expectant + this.tableQuery.adjustStartKey(currentStartKey); + setExpectant(refreshPartition(tableQuery, tableName)); + + // Reset the iterator to start over + it = expectant.entrySet().iterator(); + referPartition.clear(); // Clear the referPartition if needed + } else { + throw e; + } + } + } + for (Map.Entry> entry : expectant.entrySet()) { // mark the refer partition referPartition.add(entry); @@ -317,9 +360,59 @@ public boolean next() throws Exception { } } + protected Map> buildPartitions(ObTableClient client, ObTableQuery tableQuery, String tableName) throws Exception { + Map> partitionObTables = new HashMap<>(); + String indexName = tableQuery.getIndexName(); + String indexTableName = null; + + if (!client.isOdpMode()) { + indexTableName = client.getIndexTableName(tableName, indexName, tableQuery.getScanRangeColumns(), false); + } + + for (ObNewRange range : tableQuery.getKeyRanges()) { + ObRowKey startKey = range.getStartKey(); + int startKeySize = startKey.getObjs().size(); + ObRowKey endKey = range.getEndKey(); + int endKeySize = endKey.getObjs().size(); + Object[] start = new Object[startKeySize]; + Object[] end = new Object[endKeySize]; + + for (int i = 0; i < startKeySize; i++) { + start[i] = startKey.getObj(i).isMinObj() || startKey.getObj(i).isMaxObj() ? + startKey.getObj(i) : startKey.getObj(i).getValue(); + } + + for (int i = 0; i < endKeySize; i++) { + end[i] = endKey.getObj(i).isMinObj() || endKey.getObj(i).isMaxObj() ? + endKey.getObj(i) : endKey.getObj(i).getValue(); + } + + ObBorderFlag borderFlag = range.getBorderFlag(); + List> pairs = client.getTables(indexTableName, + tableQuery, start, borderFlag.isInclusiveStart(), end, borderFlag.isInclusiveEnd(), + false, false); + + if (tableQuery.getScanOrder() == ObScanOrder.Reverse) { + for (int i = pairs.size() - 1; i >= 0; i--) { + partitionObTables.put(pairs.get(i).getLeft(), pairs.get(i)); + } + } else { + for (ObPair pair : pairs) { + partitionObTables.put(pair.getLeft(), pair); + } + } + } + + return partitionObTables; + } + + // 上层会不断的调getRow， 也就是不断的取出缓存，所以可以在这里给row赋值的时候顺便把这个row记录下来，用来作为最后拿到的key protected void nextRow() { rowIndex = rowIndex + 1; row = cacheRows.poll(); + if (row != null) { + currentStartKey = (ObRowKey) row.get(0).getValue(); + } } protected void checkStatus() throws IllegalStateException { @@ -397,6 +490,7 @@ protected abstract ObTableQueryResult execute(ObPair partIdW protected abstract ObTableQueryAsyncResult executeAsync(ObPair partIdWithObTable, ObPayload streamRequest) throws Exception; + protected abstract Map> refreshPartition(ObTableQuery tableQuery, String tableName) throws Exception; protected void cacheResultRows(ObTableQueryResult tableQueryResult) { cacheRows.addAll(tableQueryResult.getPropertiesRows()); diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java index 56bdd2d6..de4d9dc0 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java @@ -21,6 +21,8 @@ import com.alipay.oceanbase.rpc.table.ObHBaseParams; import com.alipay.oceanbase.rpc.table.ObKVParams; import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation.ObTableAggregationSingle; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation.ObTableAggregationType; import com.alipay.oceanbase.rpc.util.Serialization; @@ -69,6 +71,86 @@ public class ObTableQuery extends AbstractPayload { private ObKVParams obKVParams; + public void adjustStartKey(ObRowKey key) throws IllegalArgumentException { + List keyRanges = getKeyRanges(); + for (ObNewRange range : keyRanges) { + if (isKeyInRange(range, key)) { + byte[] bytes = parseStartKeyToBytes(key); + ObRowKey newStartKey = ObRowKey.getInstance(new Object[]{incrementByteArray(bytes), ObObj.getMin(), ObObj.getMin()}); + range.setStartKey(newStartKey); + return; + } + } + throw new IllegalArgumentException("Key not found in any KeyRange."); + } + + private byte[] parseStartKeyToBytes(ObRowKey key) { + if (key == null || key.getObjs() == null || key.getObjs().isEmpty()) { + return new byte[0]; + } + + ObObj obObjKey = key.getObjs().get(0); + if (obObjKey.getValue() instanceof byte[]) { + return (byte[]) obObjKey.getValue(); + } + + throw new IllegalArgumentException("The start key does not contain a byte[] value."); + } + + private boolean isKeyInRange(ObNewRange range, ObRowKey key) { + byte[] startKeyBytes = parseStartKeyToBytes(range.getStartKey()); + byte[] endKeyBytes = parseStartKeyToBytes(range.getEndKey()); + byte[] keyBytes = parseStartKeyToBytes(key); + + int startComparison = compareByteArrays(startKeyBytes, keyBytes); + int endComparison = compareByteArrays(endKeyBytes, keyBytes); + + // 假设范围是 [start, end)，那么需要： + boolean withinStart = startComparison <= 0; // key >= start + boolean withinEnd = endComparison > 0; // key < end + + return withinStart && withinEnd; + } + + private int compareByteArrays(byte[] array1, byte[] array2) { + for (int i = 0; i < Math.min(array1.length, array2.length); i++) { + int a = (array1[i] & 0xFF); + int b = (array2[i] & 0xFF); + if (a != b) { + return a - b; + } + } + return array1.length - array2.length; + } + + private byte[] incrementByteArray(byte[] input) { + if (input == null || input.length == 0) { + return new byte[]{1}; // 仅处理空输入情况 + } + + byte[] result = input.clone(); // 复制数组以保持原数据不变 + + for (int i = result.length - 1; i >= 0; i--) { + // 增加该字节1 + result[i] += 1; + + // 检查是否出现溢出 + if ((result[i] & 0xFF) != 0) { + // 没有溢出，直接返回结果，不必再加了 + return result; + } + + // 若溢出，需要下一个字节处理进位（当前字节已0） + result[i] = 0; + } + + // 如果循环结束还没有返回，说明最高位有进位 + byte[] extendedResult = new byte[result.length + 1]; + extendedResult[0] = 1; // 设置第一个字节，代表进位 + + return extendedResult; + } + /* * Check filter. */ diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java index 79096c31..ed4e88c5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java @@ -20,12 +20,11 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.bolt.transport.ObTableConnection; import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.exception.ObTableNeedFetchAllException; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.protocol.payload.Constants; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.AbstractQueryStreamResult; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryRequest; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObQueryOperationType; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncResult; @@ -145,6 +144,11 @@ protected void closeLastStreamResult(ObPair partIdWithObTabl } } + @Override + protected Map> refreshPartition(ObTableQuery tableQuery, String tableName) throws Exception { + return buildPartitions(client, tableQuery, tableName); + } + @Override public boolean next() throws Exception { checkStatus(); @@ -180,8 +184,18 @@ public boolean next() throws Exception { .iterator(); while (it.hasNext()) { Map.Entry> entry = it.next(); - // try access new partition, async will not remove useless expectant - referToNewPartition(entry.getValue()); + try { + // try access new partition, async will not remove useless expectant + referToNewPartition(entry.getValue()); + } catch (Exception e) { + if (e instanceof ObTableNeedFetchAllException) { + this.tableQuery.adjustStartKey(currentStartKey); + setExpectant(refreshPartition(tableQuery, tableName)); + it = expectant.entrySet().iterator(); + } else { + throw e; + } + } // remove useless expectant if it is end if (isEnd()) diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java index ef55baeb..9b8dc4f3 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java @@ -22,6 +22,7 @@ import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.AbstractQueryStreamResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQuery; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncResult; @@ -29,6 +30,7 @@ import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; import org.slf4j.Logger; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; public class ObTableClientQueryStreamResult extends AbstractQueryStreamResult { @@ -76,6 +78,11 @@ protected ObTableQueryAsyncResult executeAsync(ObPair partId throw new IllegalArgumentException("not support this execute"); } + @Override + protected Map> refreshPartition(ObTableQuery tableQuery, String tableName) throws Exception { + return buildPartitions(client, tableQuery, tableName); + } + /** * Get client. * @return client diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java index 145fbbbf..46cf35a8 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java @@ -32,6 +32,7 @@ import com.alipay.oceanbase.rpc.table.api.TableQuery; import com.alipay.oceanbase.rpc.util.MonitorUtil; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -240,52 +241,57 @@ ObTableClientQueryAsyncStreamResult execute() throws Exception { }); } - /* - * Init partition tables involved in this query - */ - public void initPartitions() throws Exception { + public Map> initPartitions(ObTableQuery tableQuery, String tableName) throws Exception { + Map> partitionObTables = new HashMap<>(); String indexName = tableQuery.getIndexName(); + String indexTableName = null; + if (!this.obTableClient.isOdpMode()) { - indexTableName = obTableClient.getIndexTableName(tableName, indexName, - tableQuery.getScanRangeColumns(), false); + indexTableName = obTableClient.getIndexTableName(tableName, indexName, tableQuery.getScanRangeColumns(), false); } - for (ObNewRange rang : this.tableQuery.getKeyRanges()) { - ObRowKey startKey = rang.getStartKey(); + for (ObNewRange range : tableQuery.getKeyRanges()) { + ObRowKey startKey = range.getStartKey(); int startKeySize = startKey.getObjs().size(); - ObRowKey endKey = rang.getEndKey(); + ObRowKey endKey = range.getEndKey(); int endKeySize = endKey.getObjs().size(); Object[] start = new Object[startKeySize]; Object[] end = new Object[endKeySize]; + for (int i = 0; i < startKeySize; i++) { - if (startKey.getObj(i).isMinObj() || startKey.getObj(i).isMaxObj()) { - start[i] = startKey.getObj(i); - } else { - start[i] = startKey.getObj(i).getValue(); - } + start[i] = startKey.getObj(i).isMinObj() || startKey.getObj(i).isMaxObj() ? + startKey.getObj(i) : startKey.getObj(i).getValue(); } + for (int i = 0; i < endKeySize; i++) { - if (endKey.getObj(i).isMinObj() || endKey.getObj(i).isMaxObj()) { - end[i] = endKey.getObj(i); - } else { - end[i] = endKey.getObj(i).getValue(); - } + end[i] = endKey.getObj(i).isMinObj() || endKey.getObj(i).isMaxObj() ? + endKey.getObj(i) : endKey.getObj(i).getValue(); } - ObBorderFlag borderFlag = rang.getBorderFlag(); - // pairs -> List> + + ObBorderFlag borderFlag = range.getBorderFlag(); List> pairs = this.obTableClient.getTables(indexTableName, - tableQuery, start, borderFlag.isInclusiveStart(), end, borderFlag.isInclusiveEnd(), - false, false); - if (this.tableQuery.getScanOrder() == ObScanOrder.Reverse) { + tableQuery, start, borderFlag.isInclusiveStart(), end, borderFlag.isInclusiveEnd(), + false, false); + + if (tableQuery.getScanOrder() == ObScanOrder.Reverse) { for (int i = pairs.size() - 1; i >= 0; i--) { - this.partitionObTables.put(pairs.get(i).getLeft(), pairs.get(i)); + partitionObTables.put(pairs.get(i).getLeft(), pairs.get(i)); } } else { for (ObPair pair : pairs) { - this.partitionObTables.put(pair.getLeft(), pair); + partitionObTables.put(pair.getLeft(), pair); } } } + + return partitionObTables; + } + + /* + * Init partition tables involved in this query + */ + public void initPartitions() throws Exception { + this.partitionObTables = initPartitions(tableQuery, tableName); } /* From f0de20dec39bff2e6d607a0d8ecf6afdf32cb51d Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 12 Aug 2024 15:08:44 +0800 Subject: [PATCH 07/23] fix(scan): prevent infinite retries on illegal partitions during splits --- .../query/AbstractQueryStreamResult.java | 19 ++- .../impl/execute/query/ObTableQuery.java | 80 +++-------- .../ObTableClientQueryAsyncStreamResult.java | 7 +- .../ObTableClientQueryStreamResult.java | 6 +- .../table/ObTableClientLSBatchOpsImpl.java | 2 +- .../alipay/oceanbase/rpc/util/ByteUtil.java | 132 ++++++++++++++++++ 6 files changed, 174 insertions(+), 72 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/rpc/util/ByteUtil.java diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java index 45887026..0d7c5ff3 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java @@ -65,7 +65,7 @@ public abstract class AbstractQueryStreamResult extends AbstractPayload implemen protected LinkedList> cacheRows = new LinkedList>(); private LinkedList, ObTableQueryResult>> partitionLastResult = new LinkedList, ObTableQueryResult>>(); private ObReadConsistency readConsistency = ObReadConsistency.STRONG; - public ObRowKey currentStartKey; + public List currentStartKey; /* * Get pcode. @@ -302,7 +302,8 @@ public boolean next() throws Exception { // lastly, refer to the new partition boolean hasNext = false; List>> referPartition = new ArrayList>>(); - Iterator>> it = expectant.entrySet().iterator(); + Iterator>> it = expectant.entrySet() + .iterator(); while (it.hasNext()) { Map.Entry> entry = it.next(); referPartition.add(entry); @@ -311,7 +312,8 @@ public boolean next() throws Exception { referPartition.add(entry); // Try accessing the new partition - ObTableQueryResult tableQueryResult = (ObTableQueryResult) referToNewPartition(entry.getValue()); + ObTableQueryResult tableQueryResult = (ObTableQueryResult) referToNewPartition(entry + .getValue()); if (tableQueryResult.getRowCount() == 0) { continue; @@ -329,13 +331,13 @@ public boolean next() throws Exception { // Reset the iterator to start over it = expectant.entrySet().iterator(); - referPartition.clear(); // Clear the referPartition if needed + referPartition.clear(); // Clear the referPartition if needed } else { throw e; } } } - + for (Map.Entry> entry : expectant.entrySet()) { // mark the refer partition referPartition.add(entry); @@ -411,7 +413,7 @@ protected void nextRow() { rowIndex = rowIndex + 1; row = cacheRows.poll(); if (row != null) { - currentStartKey = (ObRowKey) row.get(0).getValue(); + currentStartKey = row; } } @@ -490,7 +492,10 @@ protected abstract ObTableQueryResult execute(ObPair partIdW protected abstract ObTableQueryAsyncResult executeAsync(ObPair partIdWithObTable, ObPayload streamRequest) throws Exception; - protected abstract Map> refreshPartition(ObTableQuery tableQuery, String tableName) throws Exception; + + protected abstract Map> refreshPartition(ObTableQuery tableQuery, + String tableName) + throws Exception; protected void cacheResultRows(ObTableQueryResult tableQueryResult) { cacheRows.addAll(tableQueryResult.getPropertiesRows()); diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java index de4d9dc0..ec0bf9e0 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java @@ -28,9 +28,11 @@ import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; +import static com.alipay.oceanbase.rpc.util.ByteUtil.*; import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; @@ -70,86 +72,44 @@ public class ObTableQuery extends AbstractPayload { private List aggregations = new LinkedList<>(); private ObKVParams obKVParams; - - public void adjustStartKey(ObRowKey key) throws IllegalArgumentException { + + public void adjustStartKey(List key) throws IllegalArgumentException { List keyRanges = getKeyRanges(); for (ObNewRange range : keyRanges) { if (isKeyInRange(range, key)) { byte[] bytes = parseStartKeyToBytes(key); - ObRowKey newStartKey = ObRowKey.getInstance(new Object[]{incrementByteArray(bytes), ObObj.getMin(), ObObj.getMin()}); - range.setStartKey(newStartKey); + if (getScanOrder() == ObScanOrder.Forward) { + ObRowKey newStartKey = ObRowKey.getInstance(new Object[]{incrementByteArray(bytes), ObObj.getMin(), ObObj.getMin()}); + range.setStartKey(newStartKey); + } else { + ObRowKey newStartKey = ObRowKey.getInstance(new Object[]{decrementByteArray(bytes), ObObj.getMax(), ObObj.getMax()}); + range.setEndKey(newStartKey); + } return; } } throw new IllegalArgumentException("Key not found in any KeyRange."); } - private byte[] parseStartKeyToBytes(ObRowKey key) { - if (key == null || key.getObjs() == null || key.getObjs().isEmpty()) { - return new byte[0]; - } - - ObObj obObjKey = key.getObjs().get(0); - if (obObjKey.getValue() instanceof byte[]) { - return (byte[]) obObjKey.getValue(); - } - - throw new IllegalArgumentException("The start key does not contain a byte[] value."); + private byte[] parseStartKeyToBytes(List key) { + ObObj obObjKey = key.get(0); + return obObjKey.encode(); } - private boolean isKeyInRange(ObNewRange range, ObRowKey key) { - byte[] startKeyBytes = parseStartKeyToBytes(range.getStartKey()); - byte[] endKeyBytes = parseStartKeyToBytes(range.getEndKey()); + private boolean isKeyInRange(ObNewRange range, List key) { + byte[] startKeyBytes = parseStartKeyToBytes(range.getStartKey().getObjs()); + byte[] endKeyBytes = parseStartKeyToBytes(range.getEndKey().getObjs()); byte[] keyBytes = parseStartKeyToBytes(key); int startComparison = compareByteArrays(startKeyBytes, keyBytes); int endComparison = compareByteArrays(endKeyBytes, keyBytes); - - // 假设范围是 [start, end)，那么需要： - boolean withinStart = startComparison <= 0; // key >= start - boolean withinEnd = endComparison > 0; // key < end + + boolean withinStart = startComparison <= 0; + boolean withinEnd = endComparison > 0; return withinStart && withinEnd; } - private int compareByteArrays(byte[] array1, byte[] array2) { - for (int i = 0; i < Math.min(array1.length, array2.length); i++) { - int a = (array1[i] & 0xFF); - int b = (array2[i] & 0xFF); - if (a != b) { - return a - b; - } - } - return array1.length - array2.length; - } - - private byte[] incrementByteArray(byte[] input) { - if (input == null || input.length == 0) { - return new byte[]{1}; // 仅处理空输入情况 - } - - byte[] result = input.clone(); // 复制数组以保持原数据不变 - - for (int i = result.length - 1; i >= 0; i--) { - // 增加该字节1 - result[i] += 1; - - // 检查是否出现溢出 - if ((result[i] & 0xFF) != 0) { - // 没有溢出，直接返回结果，不必再加了 - return result; - } - - // 若溢出，需要下一个字节处理进位（当前字节已0） - result[i] = 0; - } - - // 如果循环结束还没有返回，说明最高位有进位 - byte[] extendedResult = new byte[result.length + 1]; - extendedResult[0] = 1; // 设置第一个字节，代表进位 - - return extendedResult; - } /* * Check filter. diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java index ed4e88c5..533d0af9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java @@ -145,10 +145,12 @@ protected void closeLastStreamResult(ObPair partIdWithObTabl } @Override - protected Map> refreshPartition(ObTableQuery tableQuery, String tableName) throws Exception { + protected Map> refreshPartition(ObTableQuery tableQuery, + String tableName) + throws Exception { return buildPartitions(client, tableQuery, tableName); } - + @Override public boolean next() throws Exception { checkStatus(); @@ -192,6 +194,7 @@ public boolean next() throws Exception { this.tableQuery.adjustStartKey(currentStartKey); setExpectant(refreshPartition(tableQuery, tableName)); it = expectant.entrySet().iterator(); + continue; } else { throw e; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java index 9b8dc4f3..2c0bf482 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java @@ -79,10 +79,12 @@ protected ObTableQueryAsyncResult executeAsync(ObPair partId } @Override - protected Map> refreshPartition(ObTableQuery tableQuery, String tableName) throws Exception { + protected Map> refreshPartition(ObTableQuery tableQuery, + String tableName) + throws Exception { return buildPartitions(client, tableQuery, tableName); } - + /** * Get client. * @return client diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index 178f3fb3..b08bee14 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -675,7 +675,7 @@ public ObTableSingleOpResult[] executeInternal() throws Exception { long getTableTime = System.currentTimeMillis(); final Map context = ThreadLocalMap.getContextMap(); final int maxRetries = obTableClient.getRuntimeRetryTimes(); - + if (executorService != null && !executorService.isShutdown() && lsOperations.size() > 1) { // execute sub-batch operation in parallel final ConcurrentTaskExecutor executor = new ConcurrentTaskExecutor(executorService, diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/ByteUtil.java b/src/main/java/com/alipay/oceanbase/rpc/util/ByteUtil.java new file mode 100644 index 00000000..4a324192 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/util/ByteUtil.java @@ -0,0 +1,132 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.util; + +public class ByteUtil { + /** + * Compares two byte arrays lexicographically. + * + * This method interprets each byte as an unsigned value between 0 and 255 + * and performs a comparison on the arrays byte-by-byte. If two corresponding + * bytes differ, it returns the difference between the first non-equal byte + * values (treated as unsigned). If one array is a prefix of the other, + * the shorter array is considered to be less than the longer one. + * + * @param array1 the first byte array to be compared + * @param array2 the second byte array to be compared + * @return a negative integer if array1 comes before array2, + * a positive integer if array1 comes after array2, + * or zero if they are equal + */ + static public int compareByteArrays(byte[] array1, byte[] array2) { + for (int i = 0; i < Math.min(array1.length, array2.length); i++) { + int a = (array1[i] & 0xFF); + int b = (array2[i] & 0xFF); + if (a != b) { + return a - b; + } + } + return array1.length - array2.length; + } + + /** + * Increments a byte array treated as an unsigned integer by one. + * + * This method treats the input byte array as a non-negative big-endian + * integer, and increments its value by 1. The most significant byte is at + * the beginning of the array. If the increment operation causes an overflow, + * the resulting array will be extended to accommodate the carry. + * + * @param input the byte array representing a non-negative integer + * @return a new byte array representing the incremented value + */ + static public byte[] incrementByteArray(byte[] input) { + if (input == null || input.length == 0) { + return new byte[] { 1 }; + } + byte[] result = input.clone(); + for (int i = result.length - 1; i >= 0; i--) { + result[i] += 1; + + if ((result[i] & 0xFF) != 0) { + return result; + } + result[i] = 0; + } + + byte[] extendedResult = new byte[result.length + 1]; + extendedResult[0] = 1; + return extendedResult; + } + + /** + * Decrements a byte array treated as an unsigned integer by one. + * + * This method treats the input byte array as a non-negative big-endian + * integer, and decrements its value by 1. If the entire array represents + * zero, the function should handle it appropriately, typically returning + * an array representing zero. + * + * @param input the byte array representing a non-negative integer + * @return a new byte array representing the decremented value + * @throws IllegalArgumentException if the input represents zero + */ + static public byte[] decrementByteArray(byte[] input) { + if (input == null || input.length == 0 || isZero(input)) { + throw new IllegalArgumentException("Input array must represent a positive integer."); + } + + byte[] result = input.clone(); + + // Traverse the array from the least significant byte to the most + for (int i = result.length - 1; i >= 0; i--) { + if ((result[i] & 0xFF) > 0) { + result[i] -= 1; + return result; + } + + // If the byte is zero, we need to borrow from the next + result[i] = (byte) 0xFF; // Set current byte to 255 after borrow + } + + // Handle cases where we subtract from a number like 1000 -> 999 + if (result[0] == 0) { + byte[] shortenedResult = new byte[result.length - 1]; + System.arraycopy(result, 1, shortenedResult, 0, shortenedResult.length); + return shortenedResult; + } + + return result; + } + + /** + * Helper method to check if a byte array represents zero. + * + * @param array the byte array to check + * @return true if the byte array is zero, false otherwise + */ + static private boolean isZero(byte[] array) { + for (byte b : array) { + if (b != 0) { + return false; + } + } + return true; + } + +} \ No newline at end of file From c48dadb06358c8f6399844ae07cf6dfec0f5008d Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 12 Aug 2024 16:48:26 +0800 Subject: [PATCH 08/23] query refresh partitionTables --- .../query/AbstractQueryStreamResult.java | 5 +---- .../impl/execute/query/ObTableQuery.java | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java index 0d7c5ff3..bf2df14c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java @@ -129,7 +129,6 @@ protected ObPayload commonExecute(ObTableClient client, Logger logger, } tryTimes++; try { - // 重试时重新 getTable if (tryTimes > 1) { if (client.isOdpMode()) { subObTable = client.getOdpTable(); @@ -273,7 +272,6 @@ protected ObPayload commonExecute(ObTableClient client, Logger logger, /* * Next. */ - // 这个函数中，任何的计算结果都会填充到cacheRows public boolean next() throws Exception { checkStatus(); lock.lock(); @@ -325,7 +323,7 @@ public boolean next() throws Exception { } catch (Exception e) { if (e instanceof ObTableNeedFetchAllException) { - // Adjust the start key and refresh the expectant + // Adjust the start key and refresh the expectant this.tableQuery.adjustStartKey(currentStartKey); setExpectant(refreshPartition(tableQuery, tableName)); @@ -408,7 +406,6 @@ protected Map> buildPartitions(ObTableClient cl return partitionObTables; } - // 上层会不断的调getRow， 也就是不断的取出缓存，所以可以在这里给row赋值的时候顺便把这个row记录下来，用来作为最后拿到的key protected void nextRow() { rowIndex = rowIndex + 1; row = cacheRows.poll(); diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java index ec0bf9e0..ef7faad9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java @@ -76,24 +76,27 @@ public class ObTableQuery extends AbstractPayload { public void adjustStartKey(List key) throws IllegalArgumentException { List keyRanges = getKeyRanges(); for (ObNewRange range : keyRanges) { - if (isKeyInRange(range, key)) { + if (key != null && isKeyInRange(range, key)) { byte[] bytes = parseStartKeyToBytes(key); + ObRowKey newStartKey; if (getScanOrder() == ObScanOrder.Forward) { - ObRowKey newStartKey = ObRowKey.getInstance(new Object[]{incrementByteArray(bytes), ObObj.getMin(), ObObj.getMin()}); - range.setStartKey(newStartKey); + newStartKey = ObRowKey.getInstance(new Object[]{incrementByteArray(bytes), ObObj.getMin(), ObObj.getMin()}); } else { - ObRowKey newStartKey = ObRowKey.getInstance(new Object[]{decrementByteArray(bytes), ObObj.getMax(), ObObj.getMax()}); - range.setEndKey(newStartKey); + newStartKey = ObRowKey.getInstance(new Object[]{decrementByteArray(bytes), ObObj.getMax(), ObObj.getMax()}); } + range.setStartKey(newStartKey); return; } } - throw new IllegalArgumentException("Key not found in any KeyRange."); + /* keyRanges not changed */ } private byte[] parseStartKeyToBytes(List key) { - ObObj obObjKey = key.get(0); - return obObjKey.encode(); + if (key != null) { + ObObj obObjKey = key.get(0); + return obObjKey.encode(); + } + return new byte[0]; } private boolean isKeyInRange(ObNewRange range, List key) { From 8e0f12004982bd2a7a787bd0ef05050054451fee Mon Sep 17 00:00:00 2001 From: maochongxin Date: Thu, 15 Aug 2024 11:21:17 +0800 Subject: [PATCH 09/23] fix scan_to_next_partition_test route error; fix scan_during_split_test result error --- .../query/AbstractQueryStreamResult.java | 1 + .../impl/execute/query/ObTableQuery.java | 7 +++--- .../ObTableClientQueryAsyncStreamResult.java | 25 +++++++++++++++---- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java index bf2df14c..a11a8a67 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java @@ -65,6 +65,7 @@ public abstract class AbstractQueryStreamResult extends AbstractPayload implemen protected LinkedList> cacheRows = new LinkedList>(); private LinkedList, ObTableQueryResult>> partitionLastResult = new LinkedList, ObTableQueryResult>>(); private ObReadConsistency readConsistency = ObReadConsistency.STRONG; + // ObRowKey objs: [startKey, MIN_OBJECT, MIN_OBJECT] public List currentStartKey; /* diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java index ef7faad9..836e9e93 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java @@ -77,17 +77,18 @@ public void adjustStartKey(List key) throws IllegalArgumentException { List keyRanges = getKeyRanges(); for (ObNewRange range : keyRanges) { if (key != null && isKeyInRange(range, key)) { - byte[] bytes = parseStartKeyToBytes(key); ObRowKey newStartKey; if (getScanOrder() == ObScanOrder.Forward) { - newStartKey = ObRowKey.getInstance(new Object[]{incrementByteArray(bytes), ObObj.getMin(), ObObj.getMin()}); + // get the real rowkey + newStartKey = ObRowKey.getInstance(new Object[]{key.get(0).getValue(), ObObj.getMax(), ObObj.getMax()}); } else { - newStartKey = ObRowKey.getInstance(new Object[]{decrementByteArray(bytes), ObObj.getMax(), ObObj.getMax()}); + newStartKey = ObRowKey.getInstance(new Object[]{key.get(0).getValue(), ObObj.getMax(), ObObj.getMax()}); } range.setStartKey(newStartKey); return; } } + System.out.println("noting changed"); /* keyRanges not changed */ } diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java index 533d0af9..5d9619d1 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java @@ -29,6 +29,7 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncResult; import com.alipay.oceanbase.rpc.table.ObTableParam; +import com.alipay.oceanbase.rpc.util.ByteUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -166,10 +167,22 @@ public boolean next() throws Exception { if (!isEnd() && !expectant.isEmpty()) { Iterator>> it = expectant.entrySet() .iterator(); - Map.Entry> lastEntry = it.next(); - // try access new partition, async will not remove useless expectant - referToLastStreamResult(lastEntry.getValue()); + Map.Entry> lastEntry = it.next(); + try { + // try access new partition, async will not remove useless expectant + referToLastStreamResult(lastEntry.getValue()); + } catch (Exception e) { + if (e instanceof ObTableNeedFetchAllException) { + this.asyncRequest.getObTableQueryRequest().getTableQuery() + .adjustStartKey(currentStartKey); + setExpectant(refreshPartition(this.asyncRequest.getObTableQueryRequest() + .getTableQuery(), tableName)); + setEnd(true); + } else { + throw e; + } + } // remove useless expectant if it is end if (isEnd()) it.remove(); @@ -191,8 +204,10 @@ public boolean next() throws Exception { referToNewPartition(entry.getValue()); } catch (Exception e) { if (e instanceof ObTableNeedFetchAllException) { - this.tableQuery.adjustStartKey(currentStartKey); - setExpectant(refreshPartition(tableQuery, tableName)); + this.asyncRequest.getObTableQueryRequest().getTableQuery() + .adjustStartKey(currentStartKey); + setExpectant(refreshPartition(this.asyncRequest.getObTableQueryRequest() + .getTableQuery(), tableName)); it = expectant.entrySet().iterator(); continue; } else { From 8288ba1a198f7567021f9b6495bed6ad79ce2543 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Sat, 17 Aug 2024 22:32:08 +0800 Subject: [PATCH 10/23] fix getScanner error during split --- .../ObTableClientQueryAsyncStreamResult.java | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java index 5d9619d1..b640fa71 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java @@ -21,6 +21,7 @@ import com.alipay.oceanbase.rpc.bolt.transport.ObTableConnection; import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTableNeedFetchAllException; +import com.alipay.oceanbase.rpc.exception.ObTableRetryExhaustedException; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.protocol.payload.Constants; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; @@ -36,6 +37,8 @@ import java.util.*; import java.util.concurrent.atomic.AtomicReference; +import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.RUNTIME; + public class ObTableClientQueryAsyncStreamResult extends AbstractQueryStreamResult { private static final Logger logger = LoggerFactory .getLogger(ObTableClientQueryStreamResult.class); @@ -50,6 +53,7 @@ public void init() throws Exception { if (initialized) { return; } + int maxRetries = client.getTableEntryRefreshTryTimes(); // init request ObTableQueryRequest request = new ObTableQueryRequest(); request.setTableName(tableName); @@ -66,8 +70,29 @@ public void init() throws Exception { if (!expectant.isEmpty()) { Iterator>> it = expectant.entrySet() .iterator(); - Map.Entry> firstEntry = it.next(); - referToNewPartition(firstEntry.getValue()); + int retryTimes = 0; + while (it.hasNext()) { + Map.Entry> firstEntry = it.next(); + try { + // try access new partition, async will not remove useless expectant + referToNewPartition(firstEntry.getValue()); + break; + } catch (Exception e) { + if (e instanceof ObTableNeedFetchAllException) { + setExpectant(refreshPartition(this.asyncRequest.getObTableQueryRequest() + .getTableQuery(), tableName)); + it = expectant.entrySet().iterator(); + retryTimes++; + if (retryTimes > maxRetries) { + RUNTIME.error("Fail to get refresh table entry response after {}", retryTimes); + throw new ObTableRetryExhaustedException("Fail to get refresh table entry response after " + retryTimes); + + } + } else { + throw e; + } + } + } if (isEnd()) it.remove(); } @@ -197,6 +222,7 @@ public boolean next() throws Exception { boolean hasNext = false; Iterator>> it = expectant.entrySet() .iterator(); + int retryTimes = 0; while (it.hasNext()) { Map.Entry> entry = it.next(); try { @@ -209,6 +235,11 @@ public boolean next() throws Exception { setExpectant(refreshPartition(this.asyncRequest.getObTableQueryRequest() .getTableQuery(), tableName)); it = expectant.entrySet().iterator(); + retryTimes++; + if (retryTimes > client.getTableEntryRefreshTryTimes()) { + RUNTIME.error("Fail to get refresh table entry response after {}", retryTimes); + throw new ObTableRetryExhaustedException("Fail to get refresh table entry response after " + retryTimes); + } continue; } else { throw e; From 6bd88f698d285cae6eb78c2f4bae25ea9cab747a Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 19 Aug 2024 10:20:10 +0800 Subject: [PATCH 11/23] remove unused log --- .../rpc/protocol/payload/impl/execute/query/ObTableQuery.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java index 836e9e93..13371fcc 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java @@ -88,7 +88,6 @@ public void adjustStartKey(List key) throws IllegalArgumentException { return; } } - System.out.println("noting changed"); /* keyRanges not changed */ } From ecf8cc66aba52a4771b60106812d88536ff2710b Mon Sep 17 00:00:00 2001 From: maochongxin Date: Thu, 19 Sep 2024 19:20:42 +0800 Subject: [PATCH 12/23] Optimize SQL for refreshing table location information --- .../oceanbase/rpc/location/LocationUtil.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java index 20d657e0..8ed8165f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java @@ -154,13 +154,19 @@ public class LocationUtil { + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + "WHERE tenant_name = ? and database_name=? and table_name = ? and tablet_id = 0"; - private static final String PROXY_LOCATION_SQL_PARTITION_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.tablet_id as tablet_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " - + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type, D.ls_id as ls_id " - + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "inner join oceanbase.DBA_OB_TENANTS C on C.tenant_name = A.tenant_name " - + "left join oceanbase.CDB_OB_TABLET_TO_LS D on D.tenant_id = C.tenant_id and D.tablet_id = A.tablet_id " - + "WHERE C.tenant_name = ? and database_name= ? and table_name = ? and A.tablet_id in ({0}) "; + private static final String PROXY_LOCATION_SQL_PARTITION_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ * FROM ( " + + " SELECT A.tablet_id as tablet__id, A.svr_ip as svr_ip, A.sql_port as sql_port, A.table_id as table_id, " + + " A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, " + + " B.stop_time as stop_time, A.spare1 as replica_type " + + " FROM oceanbase.__all_virtual_proxy_schema A " + + " INNER JOIN oceanbase.__all_server B ON A.svr_ip = B.svr_ip AND A.sql_port = B.inner_port " + + " WHERE A.tablet_id IN ({0}) AND A.tenant_name = ? AND A.database_name = ? AND A.table_name = ?) AS left_table " + + "LEFT JOIN (" + + " SELECT D.ls_id, D.tablet_id " + + " FROM oceanbase.__all_virtual_tablet_to_ls D " + + " INNER JOIN oceanbase.DBA_OB_TENANTS C ON D.tenant_id = C.tenant_id " + + " WHERE C.tenant_name = ? " + + ") AS right_table ON left_table.tablet__id = right_table.tablet_id;"; private static final String PROXY_FIRST_PARTITION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_id, part_name, tablet_id, high_bound_val, sub_part_num " + "FROM oceanbase.__all_virtual_proxy_partition " @@ -797,6 +803,7 @@ public static TableEntry getTableEntryLocationFromRemote(Connection connection, ps.setString(1, key.getTenantName()); ps.setString(2, key.getDatabaseName()); ps.setString(3, key.getTableName()); + ps.setString(4, key.getTenantName()); rs = ps.executeQuery(); partitionEntry = getPartitionLocationFromResultSet(tableEntry, rs, partitionEntry); } catch (Exception e) { From a82d5d0fa962b77dab1322cae1262b59a9ecf29d Mon Sep 17 00:00:00 2001 From: maochongxin Date: Tue, 24 Sep 2024 21:03:04 +0800 Subject: [PATCH 13/23] partical refresh --- .../alipay/oceanbase/rpc/ObTableClient.java | 235 ++++++---- .../oceanbase/rpc/location/LocationUtil.java | 420 +++++++++++------- .../rpc/location/model/TableEntry.java | 26 +- .../model/partition/ObPartitionEntry.java | 39 +- .../partition/ObPartitionLocationInfo.java | 76 ++++ .../ObTableClientQueryAsyncStreamResult.java | 12 +- .../rpc/table/ObTableClientBatchOpsImpl.java | 14 +- 7 files changed, 571 insertions(+), 251 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocationInfo.java diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index efd49dc5..2677dcbf 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -1241,8 +1241,7 @@ public TableEntry getOrRefreshTableEntry(final String tableName, final boolean r // the server roster is ordered by priority long punishInterval = (long) (tableEntryRefreshIntervalBase * Math.pow(2, -serverRoster.getMaxPriority())); - punishInterval = punishInterval <= tableEntryRefreshIntervalCeiling ? punishInterval - : tableEntryRefreshIntervalCeiling; + punishInterval = Math.min(punishInterval, tableEntryRefreshIntervalCeiling); // control refresh frequency less than 100 milli second // just in case of connecting to OB Server failed or change master long interval = System.currentTimeMillis() - tableEntry.getRefreshTimeMills(); @@ -1267,8 +1266,7 @@ public TableEntry getOrRefreshTableEntry(final String tableName, final boolean r } int serverSize = serverRoster.getMembers().size(); - int refreshTryTimes = tableEntryRefreshTryTimes > serverSize ? serverSize - : tableEntryRefreshTryTimes; + int refreshTryTimes = Math.min(tableEntryRefreshTryTimes, serverSize); for (int i = 0; i < refreshTryTimes; i++) { try { @@ -1330,6 +1328,69 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName) return refreshTableEntry(tableEntry, tableName, false); } + public TableEntry refreshTableLocationByTabletId(TableEntry tableEntry, String tableName, Long tabletId) throws ObTableAuthException { + TableEntryKey tableEntryKey = new TableEntryKey(clusterName, tenantName, database, tableName); + Lock lock = null; + try { + // if table entry is null, throw exception + if (tableEntry == null) { + throw new ObTableEntryRefreshException("table entry is null, tableName=" + tableName); + } + + long lastRefreshTime = tableEntry.getTabletLocationLastRefreshTimeMills(tabletId); + long currentTime = System.currentTimeMillis(); + // 100ms + if (currentTime - lastRefreshTime < 100) { + return tableEntry; + } + Lock tempLock = new ReentrantLock(); + lock = tableEntry.refreshLockMap.putIfAbsent(tabletId, tempLock); + lock = (lock == null) ? tempLock : lock; // check the first lock + + boolean acquired = lock.tryLock(tableEntryRefreshLockTimeout, TimeUnit.MILLISECONDS); + tableEntry.setTableLocationLastRefreshTimeMills(tabletId, currentTime); + + if (!acquired) { + String errMsg = "try to lock table-entry refreshing timeout " + "dataSource:" + + dataSourceName + " ,tableName:" + tableName + ", refresh:" + + " , timeout:" + tableEntryRefreshLockTimeout + "."; + RUNTIME.error(errMsg); + throw new ObTableEntryRefreshException(errMsg); + } + tableEntry = loadTableEntryLocationWithPriority(serverRoster, // + tableEntryKey,// + tableEntry,// + tabletId, + tableEntryAcquireConnectTimeout,// + tableEntryAcquireSocketTimeout,// + serverAddressPriorityTimeout, // + serverAddressCachingTimeout, sysUA); + // prepare the table entry for weak read. + tableEntry.prepareForWeakRead(serverRoster.getServerLdcLocation()); + } catch (ObTableNotExistException | ObTableServerCacheExpiredException e) { + RUNTIME.error("refreshTableEntry meet exception", e); + throw e; + } catch (Exception e) { + RUNTIME.error(LCD.convert("01-00020"), tableEntryKey, tableEntry, e); + throw new ObTableEntryRefreshException(String.format( + "failed to get table entry key=%s original tableEntry=%s ", tableEntryKey, + tableEntry), e); + } finally { + if (lock != null) { + lock.unlock(); + } + } + tableLocations.put(tableName, tableEntry); + + tableEntryRefreshContinuousFailureCount.set(0); + if (logger.isInfoEnabled()) { + logger.info( + "refresh table entry, dataSource: {}, tableName: {}, refresh: {} key:{} entry:{} ", + dataSourceName, tableName, true, tableEntryKey, JSON.toJSON(tableEntry)); + } + return tableEntry; + } + /** * 刷新 table entry 元数据 * @param tableEntry @@ -1345,13 +1406,7 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName, bo try { // if table entry is exist we just need to refresh table locations if (tableEntry != null && !fetchAll) { - tableEntry = loadTableEntryLocationWithPriority(serverRoster, // - tableEntryKey,// - tableEntry,// - tableEntryAcquireConnectTimeout,// - tableEntryAcquireSocketTimeout,// - serverAddressPriorityTimeout, // - serverAddressCachingTimeout, sysUA); + // do nothing } else { // if table entry is not exist we should fetch partition info and table locations tableEntry = loadTableEntryWithPriority(serverRoster, // @@ -1360,7 +1415,6 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName, bo tableEntryAcquireSocketTimeout,// serverAddressPriorityTimeout,// serverAddressCachingTimeout, sysUA); - if (tableEntry.isPartitionTable()) { switch (runningMode) { case HBASE: @@ -1544,17 +1598,22 @@ private ObPair getPartitionReplica(TableEntry tableEntry, */ private ReplicaLocation getPartitionLocation(TableEntry tableEntry, long partId, ObServerRoute route) { - if (ObGlobal.obVsnMajor() >= 4 && tableEntry.isPartitionTable()) { - ObPartitionInfo partInfo = tableEntry.getPartitionInfo(); - Map tabletIdMap = partInfo.getPartTabletIdMap(); - long partIdx = tableEntry.getPartIdx(partId); - long TabletId = tabletIdMap.get(partIdx); - return tableEntry.getPartitionEntry().getPartitionLocationWithTabletId(TabletId) - .getReplica(route); - } else { - return tableEntry.getPartitionEntry().getPartitionLocationWithPartId(partId) - .getReplica(route); + long tabletId = getTabletIdByPartId(tableEntry, partId); + return tableEntry.getPartitionEntry().getPartitionLocationWithTabletId(tabletId) + .getReplica(route); + + } + + private ReplicaLocation getPartitionLocation(ObPartitionLocationInfo obPartitionLocationInfo, + ObServerRoute route) { + while (obPartitionLocationInfo.getPartitionLocation() == null) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } } + return obPartitionLocationInfo.getPartitionLocation().getReplica(route); } /** @@ -1784,11 +1843,14 @@ public ObTable addTable(ObServerAddr addr){ public ObPair getTableInternal(String tableName, TableEntry tableEntry, long partId, boolean waitForRefresh, ObServerRoute route) throws Exception { - ObPair partitionReplica = getPartitionReplica(tableEntry, partId, - route); - - ReplicaLocation replica = partitionReplica.getRight(); - + long tabletId = getTabletIdByPartId(tableEntry, partId); + ObPartitionLocationInfo obPartitionLocationInfo = tableEntry.getPartitionEntry().getPartitionInfo(tabletId); + if (!obPartitionLocationInfo.initialized.get()) { + tableEntry = refreshTableLocationByTabletId(tableEntry, tableName, tabletId); + obPartitionLocationInfo.initialized.compareAndSet(false, true); + } + + ReplicaLocation replica = getPartitionLocation(obPartitionLocationInfo, route); ObServerAddr addr = replica.getAddr(); ObTable obTable = tableRoster.get(addr); boolean addrExpired = addr.isExpired(serverAddressCachingTimeout); @@ -1801,8 +1863,9 @@ public ObPair getTableInternal(String tableName, TableEntry logger.info("server addr {} is expired, refresh tableEntry.", addr); } - tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh, false); - replica = getPartitionReplica(tableEntry, partId, route).getRight(); + tableEntry = refreshTableLocationByTabletId(tableEntry, tableName, tabletId); + obPartitionLocationInfo = tableEntry.getPartitionEntry().getPartitionInfo(tabletId); + replica = getPartitionLocation(obPartitionLocationInfo, route); addr = replica.getAddr(); obTable = tableRoster.get(addr); } @@ -1815,17 +1878,14 @@ public ObPair getTableInternal(String tableName, TableEntry ObTableParam param = new ObTableParam(obTable); param.setPartId(partId); // used in getTable(), 4.x may change the origin partId if (ObGlobal.obVsnMajor() >= 4 && tableEntry != null) { - long partIdx = tableEntry.getPartIdx(partId); - partId = tableEntry.isPartitionTable() ? tableEntry.getPartitionInfo() - .getPartTabletIdMap().get(partIdx) : partId; - param.setLsId(tableEntry.getPartitionEntry().getLsId(partId)); + param.setLsId(obPartitionLocationInfo.getTabletLsId()); } param.setTableId(tableEntry.getTableId()); - param.setPartitionId(partId); + param.setPartitionId(tabletId); addr.recordAccess(); - return new ObPair(partitionReplica.getLeft(), param); + return new ObPair<>(tabletId, param); } /** @@ -1840,39 +1900,66 @@ public ObPair getTableInternal(String tableName, TableEntry * @throws Exception */ private List> getPartitionReplica(TableEntry tableEntry, + String tableName, Row startRow, boolean startIncluded, Row endRow, boolean endIncluded, - ObServerRoute route) - throws Exception { - // non partition - List> replicas = new ArrayList>(); - if (!tableEntry.isPartitionTable() - || tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ZERO) { - replicas.add(new ObPair(0L, getPartitionLocation(tableEntry, 0L, - route))); - return replicas; - } else if (tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ONE) { - List partIds = tableEntry.getPartitionInfo().getFirstPartDesc() - .getPartIds(startRow, startIncluded, endRow, endIncluded); - for (Long partId : partIds) { - replicas.add(new ObPair(partId, getPartitionLocation( - tableEntry, partId, route))); + ObServerRoute route) throws Exception { + List> replicas = new ArrayList<>(); + + if (!tableEntry.isPartitionTable() || tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ZERO) { + long tabletId = getTabletIdByPartId(tableEntry, 0L); + ObPartitionLocationInfo locationInfo = tableEntry.getPartitionEntry().getPartitionInfo(tabletId); + if (!locationInfo.initialized.get()) { + refreshTableLocationByTabletId(tableEntry, tableName, tabletId); + locationInfo.initialized.compareAndSet(false, true); } - } else if (tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_TWO) { - List partIds = getPartitionsForLevelTwo(tableEntry, startRow, startIncluded, - endRow, endIncluded); - for (Long partId : partIds) { - replicas.add(new ObPair(partId, getPartitionLocation( - tableEntry, partId, route))); + replicas.add(new ObPair<>(tabletId, getPartitionLocation(locationInfo, route))); + return replicas; + } + + ObPartitionLevel partitionLevel = tableEntry.getPartitionInfo().getLevel(); + List partIds = getPartitionTablePartitionIds(tableEntry, startRow, startIncluded, endRow, endIncluded, partitionLevel); + + for (Long partId : partIds) { + long tabletId = getTabletIdByPartId(tableEntry, partId); + ObPartitionLocationInfo locationInfo = tableEntry.getPartitionEntry().getPartitionInfo(tabletId); + if (!locationInfo.initialized.get()) { + refreshTableLocationByTabletId(tableEntry, tableName, tabletId); + locationInfo.initialized.compareAndSet(false, true); } + replicas.add(new ObPair<>(tabletId, getPartitionLocation(locationInfo, route))); + } + + return replicas; + } + + private List getPartitionTablePartitionIds(TableEntry tableEntry, + Row startRow, boolean startIncluded, + Row endRow, boolean endIncluded, + ObPartitionLevel level) + throws Exception { + if (level == ObPartitionLevel.LEVEL_ONE) { + return tableEntry.getPartitionInfo().getFirstPartDesc() + .getPartIds(startRow, startIncluded, endRow, endIncluded); + } else if (level == ObPartitionLevel.LEVEL_TWO) { + return getPartitionsForLevelTwo(tableEntry, startRow, startIncluded, + endRow, endIncluded); } else { RUNTIME.error("not allowed bigger than level two"); throw new ObTableGetException("not allowed bigger than level two"); } + } - return replicas; + private long getTabletIdByPartId(TableEntry tableEntry, Long partId) { + if (ObGlobal.obVsnMajor() >= 4 && tableEntry.isPartitionTable()) { + ObPartitionInfo partInfo = tableEntry.getPartitionInfo(); + Map tabletIdMap = partInfo.getPartTabletIdMap(); + long partIdx = tableEntry.getPartIdx(partId); + return tabletIdMap.getOrDefault(partIdx, partId); + } + return partId; } /** @@ -1947,28 +2034,28 @@ public List> getTables(String tableName, ObTableQuery } } - List> partIdWithReplicaList = getPartitionReplica(tableEntry, + List> partIdWithReplicaList = getPartitionReplica(tableEntry, tableName, startRow, startInclusive, endRow, endInclusive, route); // obTableParams -> List> List> obTableParams = new ArrayList>(); for (ObPair partIdWithReplica : partIdWithReplicaList) { - long partId = partIdWithReplica.getLeft(); + long tabletId = partIdWithReplica.getLeft(); ReplicaLocation replica = partIdWithReplica.getRight(); ObServerAddr addr = replica.getAddr(); ObTable obTable = tableRoster.get(addr); boolean addrExpired = addr.isExpired(serverAddressCachingTimeout); - if (addrExpired || obTable == null) { - logger - .warn( - "server address {} is expired={} or can not get ob table. So that will sync refresh metadata", - addr, addrExpired); - syncRefreshMetadata(); - tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh, false); - replica = getPartitionLocation(tableEntry, partId, route); - addr = replica.getAddr(); - obTable = tableRoster.get(addr); - } +// if (addrExpired || obTable == null) { +// logger +// .warn( +// "server address {} is expired={} or can not get ob table. So that will sync refresh metadata", +// addr, addrExpired); +// syncRefreshMetadata(); +// tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh, false); +// replica = getPartitionLocation(tableEntry, partId, route); +// addr = replica.getAddr(); +// obTable = tableRoster.get(addr); +// } if (obTable == null) { RUNTIME.error("cannot get table by addr: " + addr); @@ -1976,15 +2063,9 @@ public List> getTables(String tableName, ObTableQuery } ObTableParam param = new ObTableParam(obTable); - if (ObGlobal.obVsnMajor() >= 4) { - long partIdx = tableEntry.getPartIdx(partId); - partId = tableEntry.isPartitionTable() ? tableEntry.getPartitionInfo() - .getPartTabletIdMap().get(partIdx) : partId; - } - param.setTableId(tableEntry.getTableId()); // real partition(tablet) id - param.setPartitionId(partId); + param.setPartitionId(tabletId); addr.recordAccess(); obTableParams.add(new ObPair(partIdWithReplica.getLeft(), param)); diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java index 8ed8165f..2fa12ab0 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java @@ -56,141 +56,156 @@ public class LocationUtil { - private static final Logger logger = TableClientLoggerFactory - .getLogger(LocationUtil.class); + private static final Logger logger = TableClientLoggerFactory + .getLogger(LocationUtil.class); static { ParserConfig.getGlobalInstance().setSafeMode(true); } - private static final String OB_VERSION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ OB_VERSION() AS CLUSTER_VERSION;"; + private static final String OB_VERSION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ OB_VERSION() AS CLUSTER_VERSION;"; - private static final String PROXY_INDEX_INFO_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ data_table_id, table_id, index_type FROM oceanbase.__all_virtual_table " - + "where table_name = ?"; + private static final String PROXY_INDEX_INFO_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ data_table_id, table_id, index_type FROM oceanbase.__all_virtual_table " + + "where table_name = ?"; - private static final String PROXY_TABLE_ID_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ table_id from oceanbase.__all_virtual_proxy_schema " - + "where tenant_name = ? and database_name = ? and table_name = ? limit 1"; + private static final String PROXY_TABLE_ID_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ table_id from oceanbase.__all_virtual_proxy_schema " + + "where tenant_name = ? and database_name = ? and table_name = ? limit 1"; - private static final String OB_TENANT_EXIST_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ tenant_id from __all_tenant where tenant_name = ?;"; + private static final String OB_TENANT_EXIST_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ tenant_id from __all_tenant where tenant_name = ?;"; @Deprecated @SuppressWarnings("unused") - private static final String PROXY_PLAIN_SCHEMA_SQL_FORMAT = "SELECT /*+READ_CONSISTENCY(WEAK)*/ partition_id, svr_ip, sql_port, table_id, role, part_num, replica_num, schema_version, spare1 " - + "FROM oceanbase.__all_virtual_proxy_schema " - + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND partition_id in ({0}) AND sql_port > 0 " - + "ORDER BY role ASC LIMIT ?"; - - private static final String PROXY_PART_INFO_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_level, part_num, part_type, part_space, part_expr, " - + "part_range_type, part_interval_bin, interval_start_bin, " - + "sub_part_num, sub_part_type, sub_part_space, " - + "sub_part_range_type, def_sub_part_interval_bin, def_sub_interval_start_bin, sub_part_expr, " - + "part_key_name, part_key_type, part_key_idx, part_key_extra, spare1 " - + "FROM oceanbase.__all_virtual_proxy_partition_info " - + "WHERE table_id = ? group by part_key_name order by part_key_name LIMIT ?;"; + private static final String PROXY_PLAIN_SCHEMA_SQL_FORMAT = "SELECT /*+READ_CONSISTENCY(WEAK)*/ partition_id, svr_ip, sql_port, table_id, role, part_num, replica_num, schema_version, spare1 " + + "FROM oceanbase.__all_virtual_proxy_schema " + + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND partition_id in ({0}) AND sql_port > 0 " + + "ORDER BY role ASC LIMIT ?"; + + private static final String PROXY_PART_INFO_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_level, part_num, part_type, part_space, part_expr, " + + "part_range_type, part_interval_bin, interval_start_bin, " + + "sub_part_num, sub_part_type, sub_part_space, " + + "sub_part_range_type, def_sub_part_interval_bin, def_sub_interval_start_bin, sub_part_expr, " + + "part_key_name, part_key_type, part_key_idx, part_key_extra, spare1 " + + "FROM oceanbase.__all_virtual_proxy_partition_info " + + "WHERE table_id = ? group by part_key_name order by part_key_name LIMIT ?;"; @Deprecated @SuppressWarnings("unused") - private static final String PROXY_TENANT_SCHEMA_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ svr_ip, sql_port, table_id, role, part_num, replica_num, spare1 " - + "FROM oceanbase.__all_virtual_proxy_schema " - + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND sql_port > 0 " - + "ORDER BY partition_id ASC, role ASC LIMIT ?"; - - private static final String PROXY_DUMMY_LOCATION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " - + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type " - + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "WHERE tenant_name = ? and database_name=? and table_name = ?"; - - private static final String PROXY_LOCATION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " - + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type " - + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "WHERE tenant_name = ? and database_name=? and table_name = ? and partition_id = 0"; - - private static final String PROXY_LOCATION_SQL_PARTITION = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " - + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type " - + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "WHERE tenant_name = ? and database_name=? and table_name = ? and partition_id in ({0})"; - - private static final String PROXY_FIRST_PARTITION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_id, part_name, high_bound_val " - + "FROM oceanbase.__all_virtual_proxy_partition " - + "WHERE table_id = ? LIMIT ?;"; - - private static final String PROXY_SUB_PARTITION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ sub_part_id, part_name, high_bound_val " - + "FROM oceanbase.__all_virtual_proxy_sub_partition " - + "WHERE table_id = ? LIMIT ?;"; - - private static final String PROXY_SERVER_STATUS_INFO = "SELECT ss.svr_ip, ss.zone, zs.region, zs.spare4 as idc " - + "FROM oceanbase.__all_virtual_proxy_server_stat ss, oceanbase.__all_virtual_zone_stat zs " - + "WHERE zs.zone = ss.zone ;"; + private static final String PROXY_TENANT_SCHEMA_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ svr_ip, sql_port, table_id, role, part_num, replica_num, spare1 " + + "FROM oceanbase.__all_virtual_proxy_schema " + + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND sql_port > 0 " + + "ORDER BY partition_id ASC, role ASC LIMIT ?"; + + private static final String PROXY_DUMMY_LOCATION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ?"; + + private static final String PROXY_LOCATION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ? and partition_id = 0"; + + private static final String PROXY_LOCATION_SQL_PARTITION = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ? and partition_id in ({0})"; + + private static final String PROXY_FIRST_PARTITION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_id, part_name, high_bound_val " + + "FROM oceanbase.__all_virtual_proxy_partition " + + "WHERE table_id = ? LIMIT ?;"; + + private static final String PROXY_SUB_PARTITION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ sub_part_id, part_name, high_bound_val " + + "FROM oceanbase.__all_virtual_proxy_sub_partition " + + "WHERE table_id = ? LIMIT ?;"; + + private static final String PROXY_SERVER_STATUS_INFO = "SELECT ss.svr_ip, ss.zone, zs.region, zs.spare4 as idc " + + "FROM oceanbase.__all_virtual_proxy_server_stat ss, oceanbase.__all_virtual_zone_stat zs " + + "WHERE zs.zone = ss.zone ;"; @Deprecated @SuppressWarnings("unused") - private static final String PROXY_PLAIN_SCHEMA_SQL_FORMAT_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, schema_version, spare1 " - + "FROM oceanbase.__all_virtual_proxy_schema " - + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND tablet_id in ({0}) AND sql_port > 0 " - + "ORDER BY role ASC LIMIT ?"; - - private static final String PROXY_PART_INFO_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_level, part_num, part_type, part_space, part_expr, " - + "part_range_type, sub_part_num, sub_part_type, sub_part_space, sub_part_range_type, sub_part_expr, " - + "part_key_name, part_key_type, part_key_idx, part_key_extra, part_key_collation_type " - + "FROM oceanbase.__all_virtual_proxy_partition_info " - + "WHERE tenant_name = ? and table_id = ? group by part_key_name order by part_key_name LIMIT ?;"; + private static final String PROXY_PLAIN_SCHEMA_SQL_FORMAT_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, schema_version, spare1 " + + "FROM oceanbase.__all_virtual_proxy_schema " + + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND tablet_id in ({0}) AND sql_port > 0 " + + "ORDER BY role ASC LIMIT ?"; + + private static final String PROXY_PART_INFO_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_level, part_num, part_type, part_space, part_expr, " + + "part_range_type, sub_part_num, sub_part_type, sub_part_space, sub_part_range_type, sub_part_expr, " + + "part_key_name, part_key_type, part_key_idx, part_key_extra, part_key_collation_type " + + "FROM oceanbase.__all_virtual_proxy_partition_info " + + "WHERE tenant_name = ? and table_id = ? group by part_key_name order by part_key_name LIMIT ?;"; @Deprecated @SuppressWarnings("unused") - private static final String PROXY_TENANT_SCHEMA_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ svr_ip, sql_port, table_id, role, part_num, replica_num, spare1 " - + "FROM oceanbase.__all_virtual_proxy_schema " - + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND sql_port > 0 " - + "ORDER BY tablet_id ASC, role ASC LIMIT ?"; - - private static final String PROXY_DUMMY_LOCATION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.tablet_id as tablet_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " - + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type " - + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "WHERE tenant_name = ? and database_name=? and table_name = ?"; - - private static final String PROXY_LOCATION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.tablet_id as tablet_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " - + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type " - + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "WHERE tenant_name = ? and database_name=? and table_name = ? and tablet_id = 0"; - - private static final String PROXY_LOCATION_SQL_PARTITION_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ * FROM ( " - + " SELECT A.tablet_id as tablet__id, A.svr_ip as svr_ip, A.sql_port as sql_port, A.table_id as table_id, " - + " A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, " - + " B.stop_time as stop_time, A.spare1 as replica_type " - + " FROM oceanbase.__all_virtual_proxy_schema A " - + " INNER JOIN oceanbase.__all_server B ON A.svr_ip = B.svr_ip AND A.sql_port = B.inner_port " - + " WHERE A.tablet_id IN ({0}) AND A.tenant_name = ? AND A.database_name = ? AND A.table_name = ?) AS left_table " - + "LEFT JOIN (" - + " SELECT D.ls_id, D.tablet_id " - + " FROM oceanbase.__all_virtual_tablet_to_ls D " - + " INNER JOIN oceanbase.DBA_OB_TENANTS C ON D.tenant_id = C.tenant_id " - + " WHERE C.tenant_name = ? " - + ") AS right_table ON left_table.tablet__id = right_table.tablet_id;"; - - private static final String PROXY_FIRST_PARTITION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_id, part_name, tablet_id, high_bound_val, sub_part_num " - + "FROM oceanbase.__all_virtual_proxy_partition " - + "WHERE tenant_name = ? and table_id = ? LIMIT ?;"; - - private static final String PROXY_SUB_PARTITION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ sub_part_id, part_name, tablet_id, high_bound_val " - + "FROM oceanbase.__all_virtual_proxy_sub_partition " - + "WHERE tenant_name = ? and table_id = ? LIMIT ?;"; - - private static final String PROXY_SERVER_STATUS_INFO_V4 = "SELECT ss.svr_ip, ss.zone, zs.region, zs.idc as idc " - + "FROM DBA_OB_SERVERS ss, DBA_OB_ZONES zs " - + "WHERE zs.zone = ss.zone ;"; - - private static final String home = System.getProperty("user.home", - "/home/admin"); - - private static final String TABLE_GROUP_GET_TABLE_NAME_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ table_name " - + "FROM oceanbase.CDB_OB_TABLEGROUP_TABLES " - + "WHERE tablegroup_name = ? and tenant_id = ? limit 1;"; - - private static final int TEMPLATE_PART_ID = -1; + private static final String PROXY_TENANT_SCHEMA_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ svr_ip, sql_port, table_id, role, part_num, replica_num, spare1 " + + "FROM oceanbase.__all_virtual_proxy_schema " + + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND sql_port > 0 " + + "ORDER BY tablet_id ASC, role ASC LIMIT ?"; + + private static final String PROXY_DUMMY_LOCATION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.tablet_id as tablet_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ?"; + + private static final String PROXY_LOCATION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.tablet_id as tablet_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ? and tablet_id = 0"; + + private static final String PROXY_LOCATION_SQL_PARTITION_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ * FROM ( " + + " SELECT A.tablet_id as tablet__id, A.svr_ip as svr_ip, A.sql_port as sql_port, A.table_id as table_id, " + + " A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, " + + " B.stop_time as stop_time, A.spare1 as replica_type " + + " FROM oceanbase.__all_virtual_proxy_schema A " + + " INNER JOIN oceanbase.__all_server B ON A.svr_ip = B.svr_ip AND A.sql_port = B.inner_port " + + " WHERE A.tablet_id IN ({0}) AND A.tenant_name = ? AND A.database_name = ? AND A.table_name = ?) AS left_table " + + "LEFT JOIN (" + + " SELECT D.ls_id, D.tablet_id " + + " FROM oceanbase.__all_virtual_tablet_to_ls D " + + " INNER JOIN oceanbase.DBA_OB_TENANTS C ON D.tenant_id = C.tenant_id " + + " WHERE C.tenant_name = ? " + + ") AS right_table ON left_table.tablet__id = right_table.tablet_id;"; + + private static final String PROXY_LOCATION_SQL_PARTITION_BY_TABLETID_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ * FROM ( " + + " SELECT A.tablet_id as tablet__id, A.svr_ip as svr_ip, A.sql_port as sql_port, A.table_id as table_id, " + + " A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, " + + " B.stop_time as stop_time, A.spare1 as replica_type " + + " FROM oceanbase.__all_virtual_proxy_schema A " + + " INNER JOIN oceanbase.__all_server B ON A.svr_ip = B.svr_ip AND A.sql_port = B.inner_port " + + " WHERE A.tablet_id = ? AND A.tenant_name = ? AND A.database_name = ? AND A.table_name = ?) AS left_table " + + "LEFT JOIN (" + + " SELECT D.ls_id, D.tablet_id " + + " FROM oceanbase.__all_virtual_tablet_to_ls D " + + " INNER JOIN oceanbase.DBA_OB_TENANTS C ON D.tenant_id = C.tenant_id " + + " WHERE C.tenant_name = ? " + + ") AS right_table ON left_table.tablet__id = right_table.tablet_id;"; + + private static final String PROXY_FIRST_PARTITION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_id, part_name, tablet_id, high_bound_val, sub_part_num " + + "FROM oceanbase.__all_virtual_proxy_partition " + + "WHERE tenant_name = ? and table_id = ? LIMIT ?;"; + + private static final String PROXY_SUB_PARTITION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ sub_part_id, part_name, tablet_id, high_bound_val " + + "FROM oceanbase.__all_virtual_proxy_sub_partition " + + "WHERE tenant_name = ? and table_id = ? LIMIT ?;"; + + private static final String PROXY_SERVER_STATUS_INFO_V4 = "SELECT ss.svr_ip, ss.zone, zs.region, zs.idc as idc " + + "FROM DBA_OB_SERVERS ss, DBA_OB_ZONES zs " + + "WHERE zs.zone = ss.zone ;"; + + private static final String home = System.getProperty( + "user.home", + "/home/admin"); + + private static final String TABLE_GROUP_GET_TABLE_NAME_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ table_name " + + "FROM oceanbase.CDB_OB_TABLEGROUP_TABLES " + + "WHERE tablegroup_name = ? and tenant_id = ? limit 1;"; + + private static final int TEMPLATE_PART_ID = -1; // limit the size of get tableEntry location from remote each time - private static final int MAX_TABLET_NUMS_EPOCH = 300; + private static final int MAX_TABLET_NUMS_EPOCH = 300; private abstract static class TableEntryRefreshWithPriorityCallback { abstract T execute(ObServerAddr obServerAddr) throws ObTableEntryRefreshException; @@ -463,6 +478,37 @@ TableEntry execute(Connection connection) }); } + /* + * Load table entry location with priority by tablet id. + */ + public static TableEntry loadTableEntryLocationWithPriority(final ServerRoster serverRoster, + final TableEntryKey key, + final TableEntry tableEntry, + final Long tabletId, + final long connectTimeout, + final long socketTimeout, + final long priorityTimeout, + final long cachingTimeout, + final ObUserAuth sysUA) + throws ObTableEntryRefreshException { + + return callTableEntryRefreshWithPriority(serverRoster, priorityTimeout, cachingTimeout, + new TableEntryRefreshWithPriorityCallback() { + @Override + TableEntry execute(ObServerAddr obServerAddr) throws ObTableEntryRefreshException { + return callTableEntryRefresh(obServerAddr, key, connectTimeout, socketTimeout, + sysUA, true, new TableEntryRefreshCallback() { + @Override + TableEntry execute(Connection connection) + throws ObTablePartitionLocationRefreshException { + return getTableEntryLocationFromRemote(connection, key, tableEntry, + tabletId); + } + }); + } + }); + } + /* * load Table Name With table Group */ @@ -689,8 +735,10 @@ private static TableEntry getTableEntryFromRemote(Connection connection, TableEn } } - // get location info - getTableEntryLocationFromRemote(connection, key, tableEntry); + // only set empty partitionEntry + ObPartitionEntry partitionEntry = new ObPartitionEntry(); + tableEntry.setPartitionEntry(partitionEntry); + tableEntry.setRefreshTimeMills(System.currentTimeMillis()); if (!initialized) { if (BOOT.isInfoEnabled()) { @@ -725,6 +773,16 @@ private static TableEntry getTableEntryFromRemote(Connection connection, TableEn return tableEntry; } + private static String genLocationSQLByTabletId() { + String sql = null; + if (ObGlobal.obVsnMajor() >= 4) { + sql = PROXY_LOCATION_SQL_PARTITION_BY_TABLETID_V4; + } else { + throw new FeatureNotSupportedException("not support ob version less than 4"); + } + return sql; + } + private static String genLocationSQLByOffset(TableEntry tableEntry, int offset, int size) { StringBuilder sb = new StringBuilder(); String sql = null; @@ -782,6 +840,45 @@ private static String genLocationSQLByOffset(TableEntry tableEntry, int offset, return sql; } + public static TableEntry getTableEntryLocationFromRemote(Connection connection, + TableEntryKey key, + TableEntry tableEntry, Long tabletId) + throws ObTablePartitionLocationRefreshException { + PreparedStatement ps = null; + ResultSet rs = null; + ObPartitionEntry partitionEntry = tableEntry.getPartitionEntry(); + String sql = genLocationSQLByTabletId(); + try { + ps = connection.prepareStatement(sql); + ps.setLong(1, tabletId); + ps.setString(2, key.getTenantName()); + ps.setString(3, key.getDatabaseName()); + ps.setString(4, key.getTableName()); + ps.setString(5, key.getTenantName()); + rs = ps.executeQuery(); + getPartitionLocationFromResultSetByTablet(tableEntry, rs, partitionEntry, tabletId); + } catch (Exception e) { + RUNTIME.error(LCD.convert("01-00010"), key, tableEntry, e); + throw new ObTablePartitionLocationRefreshException(format( + "fail to get partition location entry from remote entryKey = %s tableEntry =%s ", + key, tableEntry), e); + } finally { + try { + if (null != rs) { + rs.close(); + } + if (null != ps) { + ps.close(); + } + } catch (SQLException e) { + // ignore + } + } + // 可能不需要了 + tableEntry.setRefreshTimeMills(System.currentTimeMillis()); + return tableEntry; + } + /* * Get table entry location from remote. */ @@ -793,37 +890,6 @@ public static TableEntry getTableEntryLocationFromRemote(Connection connection, PreparedStatement ps = null; ResultSet rs = null; ObPartitionEntry partitionEntry = new ObPartitionEntry(); - long partitionNum = tableEntry.getPartitionNum(); - int epoch = (int) ((partitionNum / MAX_TABLET_NUMS_EPOCH) + 1); - for (int i = 0; i < epoch; i++) { - try { - int offset = i * MAX_TABLET_NUMS_EPOCH; - String sql = genLocationSQLByOffset(tableEntry, offset, MAX_TABLET_NUMS_EPOCH); - ps = connection.prepareStatement(sql); - ps.setString(1, key.getTenantName()); - ps.setString(2, key.getDatabaseName()); - ps.setString(3, key.getTableName()); - ps.setString(4, key.getTenantName()); - rs = ps.executeQuery(); - partitionEntry = getPartitionLocationFromResultSet(tableEntry, rs, partitionEntry); - } catch (Exception e) { - RUNTIME.error(LCD.convert("01-00010"), key, partitionNum, tableEntry, e); - throw new ObTablePartitionLocationRefreshException(format( - "fail to get partition location entry from remote entryKey = %s partNum = %d tableEntry =%s " - + "offset =%d epoch =%d", key, partitionNum, tableEntry, i, epoch), e); - } finally { - try { - if (null != rs) { - rs.close(); - } - if (null != ps) { - ps.close(); - } - } catch (SQLException e) { - // ignore - } - } - } // end for tableEntry.setPartitionEntry(partitionEntry); tableEntry.setRefreshTimeMills(System.currentTimeMillis()); return tableEntry; @@ -1088,6 +1154,62 @@ private static TableEntry getTableEntryFromResultSet(TableEntryKey key, ResultSe return entry; } + private static ObPartitionEntry getPartitionLocationFromResultSetByTablet(TableEntry tableEntry, + ResultSet rs, + ObPartitionEntry partitionEntry, + long tabletId) + throws SQLException, + ObTablePartitionLocationRefreshException { + if (partitionEntry == null || tableEntry == null) { + throw new IllegalArgumentException("partitionEntry: " + partitionEntry + + " tableEntry: " + tableEntry); + } + ObPartitionLocationInfo partitionLocationInfo = partitionEntry.getPartitionInfo(tabletId); + try { + partitionLocationInfo.rwLock.writeLock().lock(); + while (rs.next()) { + ReplicaLocation replica = buildReplicaLocation(rs); + long partitionId; + long lsId; + if (ObGlobal.obVsnMajor() >= 4) { + partitionId = rs.getLong("tablet_id"); + lsId = rs.getLong("ls_id"); + if (rs.wasNull()) { + lsId = INVALID_LS_ID; // non-partitioned table + } + partitionLocationInfo.setTabletLsId(lsId); + } else { + partitionId = rs.getLong("partition_id"); + if (tableEntry.isPartitionTable() + && null != tableEntry.getPartitionInfo().getSubPartDesc()) { + partitionId = ObPartIdCalculator.getPartIdx(partitionId, tableEntry + .getPartitionInfo().getSubPartDesc().getPartNum()); + } + } + if (!replica.isValid()) { + RUNTIME + .warn(format( + "replica is invalid, continue, replica=%s, partitionId/tabletId=%d, tableId=%d", + replica, partitionId, tableEntry.getTableId())); + continue; + } + ObPartitionLocation location = partitionLocationInfo.getPartitionLocation(); + + if (location == null) { + location = new ObPartitionLocation(); + partitionLocationInfo.setPartitionLocation(location); + } + location.addReplicaLocation(replica); + } + } finally { + partitionLocationInfo.rwLock.writeLock().unlock(); + } + // TODO: v3 + if (ObGlobal.obVsnMajor() < 4) { + } + return partitionEntry; + } + private static ObPartitionEntry getPartitionLocationFromResultSet(TableEntry tableEntry, ResultSet rs, ObPartitionEntry partitionEntry) diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java index 559c9e04..fce7a2c1 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java @@ -23,9 +23,10 @@ import com.alipay.oceanbase.rpc.location.model.partition.ObPartitionLevel; import com.alipay.oceanbase.rpc.protocol.payload.Constants; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; import static com.google.common.base.Preconditions.checkArgument; @@ -53,7 +54,26 @@ public class TableEntry { // partition location private TableEntryKey tableEntryKey = null; private volatile ObPartitionEntry partitionEntry = null; - + + // tablet id ==> refresh time + private final ConcurrentHashMap refreshTimeMap = new ConcurrentHashMap<>(); + public ConcurrentHashMap refreshLockMap = new ConcurrentHashMap<>(); + + public long getTabletLocationLastRefreshTimeMills(Long tabletId) { + return refreshTimeMap.getOrDefault(tabletId, 0L); + } + public void setTableLocationLastRefreshTimeMills(Long tabletId, Long lastRefreshTime) { + refreshTimeMap.put(tabletId, lastRefreshTime); + } + + public Lock getRefreshLock(Long tabletId) { + return refreshLockMap.get(tabletId); + } + public void setRefreshLock(Long tabletId, Lock refreshLock) { + refreshLockMap.put(tabletId, refreshLock); + } + + /* * Is valid. */ @@ -218,8 +238,6 @@ public void prepare() throws IllegalArgumentException { checkArgument(partitionInfo != null, "partition table partition info is not ready. key" + tableEntryKey); partitionInfo.prepare(); - checkArgument(partitionEntry != null, - "partition table partition entry is not ready. key" + tableEntryKey); } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java index d3e87e71..709ecd99 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java @@ -21,13 +21,30 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + public class ObPartitionEntry { private Map partitionLocation = new HashMap(); // mapping from tablet id to ls id, and the part id to tablet id mapping is in ObPartitionInfo private Map tabletLsIdMap = new HashMap<>(); - + + // tabelt id -> (PartitionLocation, LsId) + // tablet id 作为索引管理PartitionInfo 其中包含了 PartitionLocation 和LSID + // 外部会通过tablet id并发的读写ObPartitionLocationInfo + // 写的场景就是更新，读的场景是正常的请求执行，需要保证读写的安全性，更新的时候一方面是保证线程安全，另一方面还需要保证不能频繁更新 + private ConcurrentHashMap partitionInfos = new ConcurrentHashMap<>(); + + + public ObPartitionLocationInfo getPartitionInfo(long tabletId) { + if (!partitionInfos.containsKey(tabletId)) { + ObPartitionLocationInfo partitionInfo = new ObPartitionLocationInfo(); + partitionInfos.put(tabletId, partitionInfo); + } + return partitionInfos.get(tabletId); + } + public Map getPartitionLocation() { return partitionLocation; } @@ -39,6 +56,16 @@ public void setPartitionLocation(Map partitionLocatio this.partitionLocation = partitionLocation; } + public Map getTabletLsIdMap() { + return tabletLsIdMap; + } + + public void setTabletLsIdMap(Map tabletLsIdMap) { + this.tabletLsIdMap = tabletLsIdMap; + } + + public long getLsId(long tabletId) { return tabletLsIdMap.get(tabletId); } + /* * Get partition location with part id. */ @@ -86,14 +113,4 @@ public void prepareForWeakRead(ObServerLdcLocation ldcLocation) { public String toString() { return "ObPartitionEntry{" + "partitionLocation=" + partitionLocation + '}'; } - - public Map getTabletLsIdMap() { - return tabletLsIdMap; - } - - public void setTabletLsIdMap(Map tabletLsIdMap) { - this.tabletLsIdMap = tabletLsIdMap; - } - - public long getLsId(long tabletId) { return tabletLsIdMap.get(tabletId); } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocationInfo.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocationInfo.java new file mode 100644 index 00000000..a226e1d4 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocationInfo.java @@ -0,0 +1,76 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.location.model.partition; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static com.alipay.oceanbase.rpc.protocol.payload.Constants.OB_INVALID_ID; + +// 这个类不做线程安全之类的处理 +public class ObPartitionLocationInfo { + private ObPartitionLocation partitionLocation = null; + private Long tabletLsId = OB_INVALID_ID; + private Long lastUpdateTime; // 最后更新时间 + public ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); // 读写锁 + public AtomicBoolean initialized = new AtomicBoolean(false); + + public ObPartitionLocationInfo() { + this.lastUpdateTime = System.currentTimeMillis(); // 初始化为当前时间 + } + + public ObPartitionLocation getPartitionLocation() { + rwLock.readLock().lock(); + try { + return partitionLocation; + } finally { + rwLock.readLock().unlock(); + } + } + + public void setPartitionLocation(ObPartitionLocation partitionLocation) { + this.partitionLocation = partitionLocation; + } + + public void updateLocation(ObPartitionLocation newLocation) { + rwLock.writeLock().lock(); + try { + this.partitionLocation = newLocation; + this.lastUpdateTime = System.currentTimeMillis(); + } finally { + rwLock.writeLock().unlock(); + } + } + + public Long getTabletLsId() { + return tabletLsId; + } + + public void setTabletLsId(Long tabletLsId) { + this.tabletLsId = tabletLsId; + } + + public Long getLastUpdateTime() { + rwLock.readLock().lock(); + try { + return lastUpdateTime; + } finally { + rwLock.readLock().unlock(); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java index b640fa71..d54c65a1 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java @@ -84,8 +84,10 @@ public void init() throws Exception { it = expectant.entrySet().iterator(); retryTimes++; if (retryTimes > maxRetries) { - RUNTIME.error("Fail to get refresh table entry response after {}", retryTimes); - throw new ObTableRetryExhaustedException("Fail to get refresh table entry response after " + retryTimes); + RUNTIME.error("Fail to get refresh table entry response after {}", + retryTimes); + throw new ObTableRetryExhaustedException( + "Fail to get refresh table entry response after " + retryTimes); } } else { @@ -237,8 +239,10 @@ public boolean next() throws Exception { it = expectant.entrySet().iterator(); retryTimes++; if (retryTimes > client.getTableEntryRefreshTryTimes()) { - RUNTIME.error("Fail to get refresh table entry response after {}", retryTimes); - throw new ObTableRetryExhaustedException("Fail to get refresh table entry response after " + retryTimes); + RUNTIME.error("Fail to get refresh table entry response after {}", + retryTimes); + throw new ObTableRetryExhaustedException( + "Fail to get refresh table entry response after " + retryTimes); } continue; } else { diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java index 5334eece..5672731a 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java @@ -20,6 +20,7 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.exception.*; import com.alipay.oceanbase.rpc.location.model.ObServerRoute; +import com.alipay.oceanbase.rpc.location.model.TableEntry; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.mutation.result.*; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; @@ -352,11 +353,12 @@ public void partitionExecute(ObTableOperationResult[] results, if (failedServerList != null) { route.setBlackList(failedServerList); } - ObTableParam newParam = obTableClient.getTableWithPartId(tableName, - originPartId, needRefreshTableEntry, - obTableClient.isTableEntryRefreshIntervalWait(), needFetchAllRouteInfo, - route).getRight(); - + TableEntry entry = obTableClient.getOrRefreshTableEntry(tableName, false, + false, false); + obTableClient.refreshTableLocationByTabletId(entry, tableName, partId); + ObTableParam newParam = obTableClient.getTableWithPartId(tableName, partId, + false, obTableClient.isTableEntryRefreshIntervalWait(), needFetchAllRouteInfo, route) + .getRight(); subObTable = newParam.getObTable(); subRequest.setPartitionId(newParam.getPartitionId()); } @@ -418,6 +420,7 @@ public void partitionExecute(ObTableOperationResult[] results, tableName, partId, ((ObTableException) ex).getErrorCode(), tryTimes, ex); if (ex instanceof ObTableNeedFetchAllException) { + // refresh table info obTableClient.getOrRefreshTableEntry(tableName, needRefreshTableEntry, obTableClient.isTableEntryRefreshIntervalWait(), true); throw ex; @@ -444,7 +447,6 @@ public void partitionExecute(ObTableOperationResult[] results, throw new ObTableUnexpectedException( "check batch operation result error: client get unexpected NULL result"); } - List subObTableOperationResults = subObTableBatchOperationResult .getResults(); From 6874323a463de563da6efa71acc208d7ec9c579b Mon Sep 17 00:00:00 2001 From: maochongxin Date: Thu, 10 Oct 2024 19:56:21 +0800 Subject: [PATCH 14/23] Fix infinite loop caused by removed reference --- .../alipay/oceanbase/rpc/ObTableClient.java | 149 ++++++++---------- .../oceanbase/rpc/location/LocationUtil.java | 47 +++--- .../model/partition/ObPartitionEntry.java | 8 +- .../partition/ObPartitionLocationInfo.java | 30 ++-- 4 files changed, 103 insertions(+), 131 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index 2677dcbf..0bd6813e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -1330,64 +1330,53 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName) public TableEntry refreshTableLocationByTabletId(TableEntry tableEntry, String tableName, Long tabletId) throws ObTableAuthException { TableEntryKey tableEntryKey = new TableEntryKey(clusterName, tenantName, database, tableName); - Lock lock = null; try { - // if table entry is null, throw exception if (tableEntry == null) { - throw new ObTableEntryRefreshException("table entry is null, tableName=" + tableName); + throw new ObTableEntryRefreshException("Table entry is null, tableName=" + tableName); } - - long lastRefreshTime = tableEntry.getTabletLocationLastRefreshTimeMills(tabletId); + + long lastRefreshTime = tableEntry.getPartitionEntry().getPartitionInfo(tabletId).getLastUpdateTime(); long currentTime = System.currentTimeMillis(); - // 100ms - if (currentTime - lastRefreshTime < 100) { + if (currentTime - lastRefreshTime < 200) { return tableEntry; } - Lock tempLock = new ReentrantLock(); - lock = tableEntry.refreshLockMap.putIfAbsent(tabletId, tempLock); - lock = (lock == null) ? tempLock : lock; // check the first lock - - boolean acquired = lock.tryLock(tableEntryRefreshLockTimeout, TimeUnit.MILLISECONDS); - tableEntry.setTableLocationLastRefreshTimeMills(tabletId, currentTime); - if (!acquired) { - String errMsg = "try to lock table-entry refreshing timeout " + "dataSource:" - + dataSourceName + " ,tableName:" + tableName + ", refresh:" + - " , timeout:" + tableEntryRefreshLockTimeout + "."; + Lock lock = tableEntry.refreshLockMap.computeIfAbsent(tabletId, k -> new ReentrantLock()); + + if (!lock.tryLock(tableEntryRefreshLockTimeout, TimeUnit.MILLISECONDS)) { + String errMsg = String.format("Try to lock table-entry refreshing timeout. DataSource: %s, TableName: %s, Timeout: %d.", + dataSourceName, tableName, tableEntryRefreshLockTimeout); RUNTIME.error(errMsg); throw new ObTableEntryRefreshException(errMsg); } - tableEntry = loadTableEntryLocationWithPriority(serverRoster, // - tableEntryKey,// - tableEntry,// - tabletId, - tableEntryAcquireConnectTimeout,// - tableEntryAcquireSocketTimeout,// - serverAddressPriorityTimeout, // - serverAddressCachingTimeout, sysUA); - // prepare the table entry for weak read. - tableEntry.prepareForWeakRead(serverRoster.getServerLdcLocation()); + + try { + tableEntry = loadTableEntryLocationWithPriority(serverRoster, tableEntryKey, tableEntry, tabletId, + tableEntryAcquireConnectTimeout, tableEntryAcquireSocketTimeout, + serverAddressPriorityTimeout, serverAddressCachingTimeout, sysUA); + + tableEntry.prepareForWeakRead(serverRoster.getServerLdcLocation()); + } finally { + lock.unlock(); + } + } catch (ObTableNotExistException | ObTableServerCacheExpiredException e) { - RUNTIME.error("refreshTableEntry meet exception", e); + RUNTIME.error("RefreshTableEntry encountered an exception", e); throw e; } catch (Exception e) { + String errorMsg = String.format("Failed to get table entry. Key=%s, Original TableEntry=%s", tableEntryKey, tableEntry); RUNTIME.error(LCD.convert("01-00020"), tableEntryKey, tableEntry, e); - throw new ObTableEntryRefreshException(String.format( - "failed to get table entry key=%s original tableEntry=%s ", tableEntryKey, - tableEntry), e); - } finally { - if (lock != null) { - lock.unlock(); - } + throw new ObTableEntryRefreshException(errorMsg, e); } - tableLocations.put(tableName, tableEntry); + tableLocations.put(tableName, tableEntry); tableEntryRefreshContinuousFailureCount.set(0); + if (logger.isInfoEnabled()) { - logger.info( - "refresh table entry, dataSource: {}, tableName: {}, refresh: {} key:{} entry:{} ", - dataSourceName, tableName, true, tableEntryKey, JSON.toJSON(tableEntry)); + logger.info("Refreshed table entry. DataSource: {}, TableName: {}, Key: {}, Entry: {}", + dataSourceName, tableName, tableEntryKey, JSON.toJSON(tableEntry)); } + return tableEntry; } @@ -1606,13 +1595,6 @@ private ReplicaLocation getPartitionLocation(TableEntry tableEntry, long partId, private ReplicaLocation getPartitionLocation(ObPartitionLocationInfo obPartitionLocationInfo, ObServerRoute route) { - while (obPartitionLocationInfo.getPartitionLocation() == null) { - try { - Thread.sleep(10); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } return obPartitionLocationInfo.getPartitionLocation().getReplica(route); } @@ -1844,48 +1826,61 @@ public ObPair getTableInternal(String tableName, TableEntry long partId, boolean waitForRefresh, ObServerRoute route) throws Exception { long tabletId = getTabletIdByPartId(tableEntry, partId); - ObPartitionLocationInfo obPartitionLocationInfo = tableEntry.getPartitionEntry().getPartitionInfo(tabletId); - if (!obPartitionLocationInfo.initialized.get()) { - tableEntry = refreshTableLocationByTabletId(tableEntry, tableName, tabletId); - obPartitionLocationInfo.initialized.compareAndSet(false, true); - } - + ObPartitionLocationInfo obPartitionLocationInfo = getOrRefreshPartitionInfo(tableEntry, tableName, tabletId); + ReplicaLocation replica = getPartitionLocation(obPartitionLocationInfo, route); ObServerAddr addr = replica.getAddr(); ObTable obTable = tableRoster.get(addr); - boolean addrExpired = addr.isExpired(serverAddressCachingTimeout); - if (obTable == null) { - logger.warn("can not get ObTable by addr {}, refresh metadata.", addr); - syncRefreshMetadata(); - } - if (addrExpired || obTable == null) { - if (logger.isInfoEnabled() && addrExpired) { - logger.info("server addr {} is expired, refresh tableEntry.", addr); - } - tableEntry = refreshTableLocationByTabletId(tableEntry, tableName, tabletId); - obPartitionLocationInfo = tableEntry.getPartitionEntry().getPartitionInfo(tabletId); + if (obTable == null || addr.isExpired(serverAddressCachingTimeout)) { + if (obTable == null) { + logger.warn("Cannot get ObTable by addr {}, refreshing metadata.", addr); + syncRefreshMetadata(); + } + if (addr.isExpired(serverAddressCachingTimeout)) { + logger.info("Server addr {} is expired, refreshing tableEntry.", addr); + } + + obPartitionLocationInfo = getOrRefreshPartitionInfo(tableEntry, tableName, tabletId); replica = getPartitionLocation(obPartitionLocationInfo, route); addr = replica.getAddr(); obTable = tableRoster.get(addr); + + if (obTable == null) { + RUNTIME.error("Cannot get table by addr: " + addr); + throw new ObTableGetException("Cannot get table by addr: " + addr); + } } - if (obTable == null) { - RUNTIME.error("cannot get table by addr: " + addr); - throw new ObTableGetException("cannot get table by addr: " + addr); + ObTableParam param = createTableParam(obTable, tableEntry, obPartitionLocationInfo, partId, tabletId); + addr.recordAccess(); + return new ObPair<>(tabletId, param); + } + + private ObPartitionLocationInfo getOrRefreshPartitionInfo(TableEntry tableEntry, + String tableName, long tabletId) + throws Exception { + ObPartitionLocationInfo obPartitionLocationInfo = tableEntry.getPartitionEntry() + .getPartitionInfo(tabletId); + if (!obPartitionLocationInfo.initialized.get()) { + tableEntry = refreshTableLocationByTabletId(tableEntry, tableName, tabletId); + obPartitionLocationInfo = tableEntry.getPartitionEntry().getPartitionInfo(tabletId); + obPartitionLocationInfo.initializationLatch.await(); } + return obPartitionLocationInfo; + } + private ObTableParam createTableParam(ObTable obTable, TableEntry tableEntry, + ObPartitionLocationInfo obPartitionLocationInfo, + long partId, long tabletId) { ObTableParam param = new ObTableParam(obTable); - param.setPartId(partId); // used in getTable(), 4.x may change the origin partId + param.setPartId(partId); if (ObGlobal.obVsnMajor() >= 4 && tableEntry != null) { param.setLsId(obPartitionLocationInfo.getTabletLsId()); } - param.setTableId(tableEntry.getTableId()); param.setPartitionId(tabletId); - - addr.recordAccess(); - return new ObPair<>(tabletId, param); + return param; } /** @@ -1910,11 +1905,7 @@ private List> getPartitionReplica(TableEntry table if (!tableEntry.isPartitionTable() || tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ZERO) { long tabletId = getTabletIdByPartId(tableEntry, 0L); - ObPartitionLocationInfo locationInfo = tableEntry.getPartitionEntry().getPartitionInfo(tabletId); - if (!locationInfo.initialized.get()) { - refreshTableLocationByTabletId(tableEntry, tableName, tabletId); - locationInfo.initialized.compareAndSet(false, true); - } + ObPartitionLocationInfo locationInfo = getOrRefreshPartitionInfo(tableEntry, tableName, tabletId); replicas.add(new ObPair<>(tabletId, getPartitionLocation(locationInfo, route))); return replicas; } @@ -1924,11 +1915,7 @@ private List> getPartitionReplica(TableEntry table for (Long partId : partIds) { long tabletId = getTabletIdByPartId(tableEntry, partId); - ObPartitionLocationInfo locationInfo = tableEntry.getPartitionEntry().getPartitionInfo(tabletId); - if (!locationInfo.initialized.get()) { - refreshTableLocationByTabletId(tableEntry, tableName, tabletId); - locationInfo.initialized.compareAndSet(false, true); - } + ObPartitionLocationInfo locationInfo = getOrRefreshPartitionInfo(tableEntry, tableName, tabletId); replicas.add(new ObPair<>(tabletId, getPartitionLocation(locationInfo, route))); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java index 2fa12ab0..6f1e3ee7 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java @@ -1160,53 +1160,54 @@ private static ObPartitionEntry getPartitionLocationFromResultSetByTablet(TableE long tabletId) throws SQLException, ObTablePartitionLocationRefreshException { + if (partitionEntry == null || tableEntry == null) { throw new IllegalArgumentException("partitionEntry: " + partitionEntry + " tableEntry: " + tableEntry); } + ObPartitionLocationInfo partitionLocationInfo = partitionEntry.getPartitionInfo(tabletId); + + partitionLocationInfo.rwLock.writeLock().lock(); try { - partitionLocationInfo.rwLock.writeLock().lock(); while (rs.next()) { ReplicaLocation replica = buildReplicaLocation(rs); - long partitionId; - long lsId; - if (ObGlobal.obVsnMajor() >= 4) { - partitionId = rs.getLong("tablet_id"); - lsId = rs.getLong("ls_id"); - if (rs.wasNull()) { - lsId = INVALID_LS_ID; // non-partitioned table - } - partitionLocationInfo.setTabletLsId(lsId); - } else { - partitionId = rs.getLong("partition_id"); - if (tableEntry.isPartitionTable() - && null != tableEntry.getPartitionInfo().getSubPartDesc()) { - partitionId = ObPartIdCalculator.getPartIdx(partitionId, tableEntry - .getPartitionInfo().getSubPartDesc().getPartNum()); - } + + long partitionId = (ObGlobal.obVsnMajor() >= 4) ? rs.getLong("tablet_id") : rs + .getLong("partition_id"); + long lsId = ObGlobal.obVsnMajor() >= 4 ? rs.getLong("ls_id") : INVALID_LS_ID; + if (rs.wasNull() && ObGlobal.obVsnMajor() >= 4) { + lsId = INVALID_LS_ID; // For non-partitioned table } + partitionLocationInfo.setTabletLsId(lsId); + + if (ObGlobal.obVsnMajor() < 4 && tableEntry.isPartitionTable() + && tableEntry.getPartitionInfo().getSubPartDesc() != null) { + partitionId = ObPartIdCalculator.getPartIdx(partitionId, tableEntry + .getPartitionInfo().getSubPartDesc().getPartNum()); + } + if (!replica.isValid()) { RUNTIME .warn(format( - "replica is invalid, continue, replica=%s, partitionId/tabletId=%d, tableId=%d", + "Replica is invalid; continuing. Replica=%s, PartitionId/TabletId=%d, TableId=%d", replica, partitionId, tableEntry.getTableId())); continue; } ObPartitionLocation location = partitionLocationInfo.getPartitionLocation(); - if (location == null) { location = new ObPartitionLocation(); - partitionLocationInfo.setPartitionLocation(location); + partitionLocationInfo.updateLocation(location); } location.addReplicaLocation(replica); + + if (partitionLocationInfo.initialized.compareAndSet(false, true)) { + partitionLocationInfo.initializationLatch.countDown(); + } } } finally { partitionLocationInfo.rwLock.writeLock().unlock(); } - // TODO: v3 - if (ObGlobal.obVsnMajor() < 4) { - } return partitionEntry; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java index 709ecd99..20af3b89 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java @@ -36,13 +36,9 @@ public class ObPartitionEntry { // 写的场景就是更新，读的场景是正常的请求执行，需要保证读写的安全性，更新的时候一方面是保证线程安全，另一方面还需要保证不能频繁更新 private ConcurrentHashMap partitionInfos = new ConcurrentHashMap<>(); - + public ObPartitionLocationInfo getPartitionInfo(long tabletId) { - if (!partitionInfos.containsKey(tabletId)) { - ObPartitionLocationInfo partitionInfo = new ObPartitionLocationInfo(); - partitionInfos.put(tabletId, partitionInfo); - } - return partitionInfos.get(tabletId); + return partitionInfos.computeIfAbsent(tabletId, id -> new ObPartitionLocationInfo()); } public Map getPartitionLocation() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocationInfo.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocationInfo.java index a226e1d4..8b9181b9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocationInfo.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocationInfo.java @@ -17,22 +17,19 @@ package com.alipay.oceanbase.rpc.location.model.partition; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; import static com.alipay.oceanbase.rpc.protocol.payload.Constants.OB_INVALID_ID; -// 这个类不做线程安全之类的处理 public class ObPartitionLocationInfo { - private ObPartitionLocation partitionLocation = null; - private Long tabletLsId = OB_INVALID_ID; - private Long lastUpdateTime; // 最后更新时间 - public ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); // 读写锁 - public AtomicBoolean initialized = new AtomicBoolean(false); - - public ObPartitionLocationInfo() { - this.lastUpdateTime = System.currentTimeMillis(); // 初始化为当前时间 - } + private ObPartitionLocation partitionLocation = null; + private Long tabletLsId = OB_INVALID_ID; + private Long lastUpdateTime = 0L; + public ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); + public AtomicBoolean initialized = new AtomicBoolean(false); + public final CountDownLatch initializationLatch = new CountDownLatch(1); public ObPartitionLocation getPartitionLocation() { rwLock.readLock().lock(); @@ -43,18 +40,9 @@ public ObPartitionLocation getPartitionLocation() { } } - public void setPartitionLocation(ObPartitionLocation partitionLocation) { - this.partitionLocation = partitionLocation; - } - public void updateLocation(ObPartitionLocation newLocation) { - rwLock.writeLock().lock(); - try { - this.partitionLocation = newLocation; - this.lastUpdateTime = System.currentTimeMillis(); - } finally { - rwLock.writeLock().unlock(); - } + this.partitionLocation = newLocation; + this.lastUpdateTime = System.currentTimeMillis(); } public Long getTabletLsId() { From 1d46a86a7e936325f07cb245c7043a95daa0ef2b Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 14 Oct 2024 16:48:14 +0800 Subject: [PATCH 15/23] fix lsop retry fail --- .../alipay/oceanbase/rpc/ObTableClient.java | 26 +++++++++---------- .../oceanbase/rpc/location/LocationUtil.java | 1 - .../table/ObTableClientLSBatchOpsImpl.java | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index 0bd6813e..3557b133 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -1364,7 +1364,7 @@ public TableEntry refreshTableLocationByTabletId(TableEntry tableEntry, String t RUNTIME.error("RefreshTableEntry encountered an exception", e); throw e; } catch (Exception e) { - String errorMsg = String.format("Failed to get table entry. Key=%s, Original TableEntry=%s", tableEntryKey, tableEntry); + String errorMsg = String.format("Failed to get table entry. Key=%s, Original TableEntry=%s, TabletId=%d", tableEntryKey, tableEntry, tabletId); RUNTIME.error(LCD.convert("01-00020"), tableEntryKey, tableEntry, e); throw new ObTableEntryRefreshException(errorMsg, e); } @@ -1939,7 +1939,7 @@ private List getPartitionTablePartitionIds(TableEntry tableEntry, } } - private long getTabletIdByPartId(TableEntry tableEntry, Long partId) { + public long getTabletIdByPartId(TableEntry tableEntry, Long partId) { if (ObGlobal.obVsnMajor() >= 4 && tableEntry.isPartitionTable()) { ObPartitionInfo partInfo = tableEntry.getPartitionInfo(); Map tabletIdMap = partInfo.getPartTabletIdMap(); @@ -2032,17 +2032,17 @@ public List> getTables(String tableName, ObTableQuery ObServerAddr addr = replica.getAddr(); ObTable obTable = tableRoster.get(addr); boolean addrExpired = addr.isExpired(serverAddressCachingTimeout); -// if (addrExpired || obTable == null) { -// logger -// .warn( -// "server address {} is expired={} or can not get ob table. So that will sync refresh metadata", -// addr, addrExpired); -// syncRefreshMetadata(); -// tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh, false); -// replica = getPartitionLocation(tableEntry, partId, route); -// addr = replica.getAddr(); -// obTable = tableRoster.get(addr); -// } + if (addrExpired || obTable == null) { + logger + .warn( + "server address {} is expired={} or can not get ob table. So that will sync refresh metadata", + addr, addrExpired); + syncRefreshMetadata(); + tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh, false); + replica = getPartitionLocation(tableEntry, tabletId, route); + addr = replica.getAddr(); + obTable = tableRoster.get(addr); + } if (obTable == null) { RUNTIME.error("cannot get table by addr: " + addr); diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java index 6f1e3ee7..9f737f83 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java @@ -874,7 +874,6 @@ public static TableEntry getTableEntryLocationFromRemote(Connection connection, // ignore } } - // 可能不需要了 tableEntry.setRefreshTimeMills(System.currentTimeMillis()); return tableEntry; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index b08bee14..9f6ad172 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -740,7 +740,7 @@ public void doTask() { // Execute sub-batch operation one by one for (final Map.Entry>>>> entry : lsOperations .entrySet()) { - partitionExecute(obTableOperationResults, entry); + executeWithRetries(obTableOperationResults, entry, maxRetries); } } From 8d60c55b0079f0cdb9e3f0bb4b258eb318f14fd2 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Tue, 15 Oct 2024 14:20:27 +0800 Subject: [PATCH 16/23] Remove unnecessary comments and format code --- .../com/alipay/oceanbase/rpc/ObTableClient.java | 13 +++++++++---- .../rpc/bolt/transport/ObTableRemoting.java | 2 +- .../rpc/location/model/TableEntry.java | 17 ----------------- .../model/partition/ObPartitionEntry.java | 3 --- .../query/AbstractQueryStreamResult.java | 1 - 5 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index 3557b133..ccb43355 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -1337,7 +1337,7 @@ public TableEntry refreshTableLocationByTabletId(TableEntry tableEntry, String t long lastRefreshTime = tableEntry.getPartitionEntry().getPartitionInfo(tabletId).getLastUpdateTime(); long currentTime = System.currentTimeMillis(); - if (currentTime - lastRefreshTime < 200) { + if (currentTime - lastRefreshTime < tableEntryRefreshIntervalBase) { return tableEntry; } @@ -2027,7 +2027,7 @@ public List> getTables(String tableName, ObTableQuery // obTableParams -> List> List> obTableParams = new ArrayList>(); for (ObPair partIdWithReplica : partIdWithReplicaList) { - long tabletId = partIdWithReplica.getLeft(); + long partId = partIdWithReplica.getLeft(); ReplicaLocation replica = partIdWithReplica.getRight(); ObServerAddr addr = replica.getAddr(); ObTable obTable = tableRoster.get(addr); @@ -2039,7 +2039,7 @@ public List> getTables(String tableName, ObTableQuery addr, addrExpired); syncRefreshMetadata(); tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh, false); - replica = getPartitionLocation(tableEntry, tabletId, route); + replica = getPartitionLocation(tableEntry, partId, route); addr = replica.getAddr(); obTable = tableRoster.get(addr); } @@ -2050,9 +2050,14 @@ public List> getTables(String tableName, ObTableQuery } ObTableParam param = new ObTableParam(obTable); + if (ObGlobal.obVsnMajor() >= 4) { + long partIdx = tableEntry.getPartIdx(partId); + partId = tableEntry.isPartitionTable() ? tableEntry.getPartitionInfo() + .getPartTabletIdMap().get(partIdx) : partId; + } param.setTableId(tableEntry.getTableId()); // real partition(tablet) id - param.setPartitionId(tabletId); + param.setPartitionId(partId); addr.recordAccess(); obTableParams.add(new ObPair(partIdWithReplica.getLeft(), param)); diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java index 4da53fb5..f7d9a200 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java @@ -122,7 +122,7 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques ObRpcResultCode resultCode = new ObRpcResultCode(); resultCode.decode(buf); // If response indicates the request is routed to wrong server, we should refresh the routing meta. - if (!conn.getObTable().getReRouting() &&response.getHeader().isRoutingWrong()) { + if (!conn.getObTable().getReRouting() && response.getHeader().isRoutingWrong()) { String errMessage = TraceUtil.formatTraceMessage(conn, request, "routed to the wrong server: " + response.getMessage()); logger.warn(errMessage); diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java index fce7a2c1..260a067c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java @@ -55,25 +55,8 @@ public class TableEntry { private TableEntryKey tableEntryKey = null; private volatile ObPartitionEntry partitionEntry = null; - // tablet id ==> refresh time - private final ConcurrentHashMap refreshTimeMap = new ConcurrentHashMap<>(); public ConcurrentHashMap refreshLockMap = new ConcurrentHashMap<>(); - public long getTabletLocationLastRefreshTimeMills(Long tabletId) { - return refreshTimeMap.getOrDefault(tabletId, 0L); - } - public void setTableLocationLastRefreshTimeMills(Long tabletId, Long lastRefreshTime) { - refreshTimeMap.put(tabletId, lastRefreshTime); - } - - public Lock getRefreshLock(Long tabletId) { - return refreshLockMap.get(tabletId); - } - public void setRefreshLock(Long tabletId, Lock refreshLock) { - refreshLockMap.put(tabletId, refreshLock); - } - - /* * Is valid. */ diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java index 20af3b89..8b514d78 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java @@ -31,9 +31,6 @@ public class ObPartitionEntry { private Map tabletLsIdMap = new HashMap<>(); // tabelt id -> (PartitionLocation, LsId) - // tablet id 作为索引管理PartitionInfo 其中包含了 PartitionLocation 和LSID - // 外部会通过tablet id并发的读写ObPartitionLocationInfo - // 写的场景就是更新，读的场景是正常的请求执行，需要保证读写的安全性，更新的时候一方面是保证线程安全，另一方面还需要保证不能频繁更新 private ConcurrentHashMap partitionInfos = new ConcurrentHashMap<>(); diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java index a11a8a67..0b846a31 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java @@ -51,7 +51,6 @@ public abstract class AbstractQueryStreamResult extends AbstractPayload implemen protected volatile boolean closed = false; protected volatile List row = null; protected volatile int rowIndex = -1; - // 调整它的startKey protected ObTableQuery tableQuery; protected long operationTimeout = -1; protected String tableName; From 22eeb36773d836ae8e5197bf16c8d4dca63334d4 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Fri, 18 Oct 2024 14:42:00 +0800 Subject: [PATCH 17/23] Fix frequent refresh lock failures due to short refresh interval --- .../java/com/alipay/oceanbase/rpc/ObTableClient.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index ccb43355..bead0ef1 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -1334,10 +1334,9 @@ public TableEntry refreshTableLocationByTabletId(TableEntry tableEntry, String t if (tableEntry == null) { throw new ObTableEntryRefreshException("Table entry is null, tableName=" + tableName); } - long lastRefreshTime = tableEntry.getPartitionEntry().getPartitionInfo(tabletId).getLastUpdateTime(); long currentTime = System.currentTimeMillis(); - if (currentTime - lastRefreshTime < tableEntryRefreshIntervalBase) { + if (currentTime - lastRefreshTime < tableEntryRefreshLockTimeout) { return tableEntry; } @@ -1364,7 +1363,7 @@ public TableEntry refreshTableLocationByTabletId(TableEntry tableEntry, String t RUNTIME.error("RefreshTableEntry encountered an exception", e); throw e; } catch (Exception e) { - String errorMsg = String.format("Failed to get table entry. Key=%s, Original TableEntry=%s, TabletId=%d", tableEntryKey, tableEntry, tabletId); + String errorMsg = String.format("Failed to get table entry. Key=%s, TabletId=%d, message=%s", tableEntryKey, tabletId, e.getMessage()); RUNTIME.error(LCD.convert("01-00020"), tableEntryKey, tableEntry, e); throw new ObTableEntryRefreshException(errorMsg, e); } @@ -2050,11 +2049,6 @@ public List> getTables(String tableName, ObTableQuery } ObTableParam param = new ObTableParam(obTable); - if (ObGlobal.obVsnMajor() >= 4) { - long partIdx = tableEntry.getPartIdx(partId); - partId = tableEntry.isPartitionTable() ? tableEntry.getPartitionInfo() - .getPartTabletIdMap().get(partIdx) : partId; - } param.setTableId(tableEntry.getTableId()); // real partition(tablet) id param.setPartitionId(partId); From 73ef9aa439bfdfd491748045d7e58b7f24cdb665 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Fri, 18 Oct 2024 15:22:23 +0800 Subject: [PATCH 18/23] Fix frequent refresh lock failures due to short refresh interval --- src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java | 5 +++++ .../oceanbase/rpc/bolt/transport/ObTableRemoting.java | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index bead0ef1..b6d7318c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -1350,6 +1350,11 @@ public TableEntry refreshTableLocationByTabletId(TableEntry tableEntry, String t } try { + lastRefreshTime = tableEntry.getPartitionEntry().getPartitionInfo(tabletId).getLastUpdateTime(); + currentTime = System.currentTimeMillis(); + if (currentTime - lastRefreshTime < tableEntryRefreshLockTimeout) { + return tableEntry; + } tableEntry = loadTableEntryLocationWithPriority(serverRoster, tableEntryKey, tableEntry, tabletId, tableEntryAcquireConnectTimeout, tableEntryAcquireSocketTimeout, serverAddressPriorityTimeout, serverAddressCachingTimeout, sysUA); diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java index f7d9a200..69f90bc9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java @@ -31,6 +31,8 @@ import io.netty.buffer.ByteBuf; import org.slf4j.Logger; +import javax.xml.transform.Result; + import static com.alipay.oceanbase.rpc.protocol.packet.ObCompressType.INVALID_COMPRESSOR; import static com.alipay.oceanbase.rpc.protocol.packet.ObCompressType.NONE_COMPRESSOR; @@ -139,7 +141,8 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques throw new ObTableNeedFetchAllException(errMessage, resultCode.getRcode()); } } - if (resultCode.getRcode() != 0 && response.getHeader().getPcode() != Pcodes.OB_TABLE_API_MOVE) { + if (resultCode.getRcode() != 0 + && response.getHeader().getPcode() != Pcodes.OB_TABLE_API_MOVE) { String errMessage = TraceUtil.formatTraceMessage(conn, request, "routed to the wrong server: " + response.getMessage()); logger.warn(errMessage); @@ -193,6 +196,7 @@ private boolean needFetchAll(int errorCode, int pcode) { || errorCode == ResultCodes.OB_TABLE_NOT_EXIST.errorCode || errorCode == ResultCodes.OB_TABLET_NOT_EXIST.errorCode || errorCode == ResultCodes.OB_LS_NOT_EXIST.errorCode + || errorCode == ResultCodes.OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST.errorCode || (pcode == Pcodes.OB_TABLE_API_LS_EXECUTE && errorCode == ResultCodes.OB_NOT_MASTER.errorCode); } From a46e2b0a7f72d98aa4028dc2326fa7a7afe9704e Mon Sep 17 00:00:00 2001 From: maochongxin Date: Fri, 18 Oct 2024 15:48:32 +0800 Subject: [PATCH 19/23] add result code -4723 --- .../java/com/alipay/oceanbase/rpc/location/LocationUtil.java | 3 ++- .../alipay/oceanbase/rpc/protocol/payload/ResultCodes.java | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java index 9f737f83..4d344f11 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java @@ -395,7 +395,8 @@ private static TableEntry callTableEntryRefresh(ObServerAddr obServerAddr, Table RUNTIME.error(LCD.convert("01-00007"), url, key, e); } throw new ObTableEntryRefreshException(format( - "fail to refresh table entry from remote url=%s, key=%s", url, key), e); + "fail to refresh table entry from remote url=%s, key=%s, message=%s", url, key, + e.getMessage()), e); } finally { try { if (null != connection) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java index fbb79d79..349912f1 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java @@ -358,7 +358,9 @@ public enum ResultCodes { OB_CLUSTER_NO_MATCH(-4666), // OB_CHECK_ZONE_MERGE_ORDER(-4667), // OB_ERR_ZONE_NOT_EMPTY(-4668), // - OB_USE_DUP_FOLLOW_AFTER_DML(-4686), OB_LS_NOT_EXIST(-4719), // + OB_USE_DUP_FOLLOW_AFTER_DML(-4686), // + OB_LS_NOT_EXIST(-4719), // + OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST(-4723), // OB_TABLET_NOT_EXIST(-4725), // OB_ERR_PARSER_INIT(-5000), // OB_ERR_PARSE_SQL(-5001), // From fb03d26c10464904c7a98ab89c6078e9072e68ee Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 23 Oct 2024 19:26:17 +0800 Subject: [PATCH 20/23] fix review --- .../alipay/oceanbase/rpc/ObTableClient.java | 50 ++++++++++++++++--- .../oceanbase/rpc/location/LocationUtil.java | 47 +++++++++++++++-- .../query/AbstractQueryStreamResult.java | 5 +- .../ObTableClientQueryAsyncStreamResult.java | 35 +++++++++---- 4 files changed, 113 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index b6d7318c..203e1679 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -1399,7 +1399,18 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName, bo try { // if table entry is exist we just need to refresh table locations if (tableEntry != null && !fetchAll) { - // do nothing + if (ObGlobal.obVsnMajor() >= 4) { + // do nothing + } else { + // 3.x still proactively refreshes all locations + tableEntry = loadTableEntryLocationWithPriority(serverRoster, // + tableEntryKey,// + tableEntry,// + tableEntryAcquireConnectTimeout,// + tableEntryAcquireSocketTimeout,// + serverAddressPriorityTimeout, // + serverAddressCachingTimeout, sysUA); + } } else { // if table entry is not exist we should fetch partition info and table locations tableEntry = loadTableEntryWithPriority(serverRoster, // @@ -1829,10 +1840,23 @@ public ObTable addTable(ObServerAddr addr){ public ObPair getTableInternal(String tableName, TableEntry tableEntry, long partId, boolean waitForRefresh, ObServerRoute route) throws Exception { + ReplicaLocation replica = null; long tabletId = getTabletIdByPartId(tableEntry, partId); - ObPartitionLocationInfo obPartitionLocationInfo = getOrRefreshPartitionInfo(tableEntry, tableName, tabletId); + ObPartitionLocationInfo obPartitionLocationInfo = null; + if (ObGlobal.obVsnMajor() >= 4) { - ReplicaLocation replica = getPartitionLocation(obPartitionLocationInfo, route); + obPartitionLocationInfo = getOrRefreshPartitionInfo(tableEntry, tableName, tabletId); + + replica = getPartitionLocation(obPartitionLocationInfo, route); + } else { + ObPair partitionReplica = getPartitionReplica(tableEntry, partId, + route); + replica = partitionReplica.getRight(); + } + if (replica == null) { + RUNTIME.error("Cannot get replica by partId: " + partId); + throw new ObTableGetException("Cannot get replica by partId: " + partId); + } ObServerAddr addr = replica.getAddr(); ObTable obTable = tableRoster.get(addr); @@ -1845,8 +1869,14 @@ public ObPair getTableInternal(String tableName, TableEntry logger.info("Server addr {} is expired, refreshing tableEntry.", addr); } - obPartitionLocationInfo = getOrRefreshPartitionInfo(tableEntry, tableName, tabletId); - replica = getPartitionLocation(obPartitionLocationInfo, route); + if (ObGlobal.obVsnMajor() >= 4) { + obPartitionLocationInfo = getOrRefreshPartitionInfo(tableEntry, tableName, tabletId); + replica = getPartitionLocation(obPartitionLocationInfo, route); + } else { + tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh, false); + replica = getPartitionReplica(tableEntry, partId, route).getRight(); + } + addr = replica.getAddr(); obTable = tableRoster.get(addr); @@ -1855,8 +1885,14 @@ public ObPair getTableInternal(String tableName, TableEntry throw new ObTableGetException("Cannot get table by addr: " + addr); } } - - ObTableParam param = createTableParam(obTable, tableEntry, obPartitionLocationInfo, partId, tabletId); + ObTableParam param = null; + if (ObGlobal.obVsnMajor() >= 4) { + param = createTableParam(obTable, tableEntry, obPartitionLocationInfo, partId, tabletId); + } else { + param.setPartId(partId); + param.setTableId(tableEntry.getTableId()); + param.setPartitionId(partId); + } addr.recordAccess(); return new ObPair<>(tabletId, param); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java index 4d344f11..ecc37f25 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java @@ -735,11 +735,16 @@ private static TableEntry getTableEntryFromRemote(Connection connection, TableEn } } } - - // only set empty partitionEntry - ObPartitionEntry partitionEntry = new ObPartitionEntry(); - tableEntry.setPartitionEntry(partitionEntry); - tableEntry.setRefreshTimeMills(System.currentTimeMillis()); + + if (ObGlobal.obVsnMajor() >= 4) { + // only set empty partitionEntry + ObPartitionEntry partitionEntry = new ObPartitionEntry(); + tableEntry.setPartitionEntry(partitionEntry); + tableEntry.setRefreshTimeMills(System.currentTimeMillis()); + } else { + // get location info + getTableEntryLocationFromRemote(connection, key, tableEntry); + } if (!initialized) { if (BOOT.isInfoEnabled()) { @@ -774,6 +779,7 @@ private static TableEntry getTableEntryFromRemote(Connection connection, TableEn return tableEntry; } + // Note: This code is applicable only for refreshing locations based on tablet ID in version 4.x private static String genLocationSQLByTabletId() { String sql = null; if (ObGlobal.obVsnMajor() >= 4) { @@ -890,6 +896,37 @@ public static TableEntry getTableEntryLocationFromRemote(Connection connection, PreparedStatement ps = null; ResultSet rs = null; ObPartitionEntry partitionEntry = new ObPartitionEntry(); + long partitionNum = tableEntry.getPartitionNum(); + int epoch = (int) ((partitionNum / MAX_TABLET_NUMS_EPOCH) + 1); + for (int i = 0; i < epoch; i++) { + try { + int offset = i * MAX_TABLET_NUMS_EPOCH; + // // This code is executed only in version 3.x + String sql = genLocationSQLByOffset(tableEntry, offset, MAX_TABLET_NUMS_EPOCH); + ps = connection.prepareStatement(sql); + ps.setString(1, key.getTenantName()); + ps.setString(2, key.getDatabaseName()); + ps.setString(3, key.getTableName()); + rs = ps.executeQuery(); + partitionEntry = getPartitionLocationFromResultSet(tableEntry, rs, partitionEntry); + } catch (Exception e) { + RUNTIME.error(LCD.convert("01-00010"), key, partitionNum, tableEntry, e); + throw new ObTablePartitionLocationRefreshException(format( + "fail to get partition location entry from remote entryKey = %s partNum = %d tableEntry =%s " + + "offset =%d epoch =%d", key, partitionNum, tableEntry, i, epoch), e); + } finally { + try { + if (null != rs) { + rs.close(); + } + if (null != ps) { + ps.close(); + } + } catch (SQLException e) { + // ignore + } + } + } // end for tableEntry.setPartitionEntry(partitionEntry); tableEntry.setRefreshTimeMills(System.currentTimeMillis()); return tableEntry; diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java index 0b846a31..d6b1711f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java @@ -17,11 +17,13 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.bolt.transport.ObTableConnection; import com.alipay.oceanbase.rpc.exception.*; import com.alipay.oceanbase.rpc.location.model.ObReadConsistency; import com.alipay.oceanbase.rpc.location.model.ObServerRoute; +import com.alipay.oceanbase.rpc.location.model.TableEntry; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; @@ -323,10 +325,7 @@ public boolean next() throws Exception { } catch (Exception e) { if (e instanceof ObTableNeedFetchAllException) { - // Adjust the start key and refresh the expectant - this.tableQuery.adjustStartKey(currentStartKey); setExpectant(refreshPartition(tableQuery, tableName)); - // Reset the iterator to start over it = expectant.entrySet().iterator(); referPartition.clear(); // Clear the referPartition if needed diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java index d54c65a1..cae7f39c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java @@ -17,11 +17,13 @@ package com.alipay.oceanbase.rpc.stream; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.bolt.transport.ObTableConnection; import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTableNeedFetchAllException; import com.alipay.oceanbase.rpc.exception.ObTableRetryExhaustedException; +import com.alipay.oceanbase.rpc.location.model.TableEntry; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.protocol.payload.Constants; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; @@ -201,11 +203,21 @@ public boolean next() throws Exception { referToLastStreamResult(lastEntry.getValue()); } catch (Exception e) { if (e instanceof ObTableNeedFetchAllException) { - this.asyncRequest.getObTableQueryRequest().getTableQuery() - .adjustStartKey(currentStartKey); - setExpectant(refreshPartition(this.asyncRequest.getObTableQueryRequest() - .getTableQuery(), tableName)); - setEnd(true); + + TableEntry entry = client.getOrRefreshTableEntry(tableName, false, false, false); + // Calculate the next partition only when the range partition is affected by a split, based on the keys already scanned. + if (ObGlobal.obVsnMajor() >= 4 + && entry.isPartitionTable() + && entry.getPartitionInfo().getFirstPartDesc().getPartFuncType().isRangePart()) { + this.asyncRequest.getObTableQueryRequest().getTableQuery() + .adjustStartKey(currentStartKey); + setExpectant(refreshPartition(this.asyncRequest.getObTableQueryRequest() + .getTableQuery(), tableName)); + setEnd(true); + } else { + setExpectant(refreshPartition(this.asyncRequest.getObTableQueryRequest() + .getTableQuery(), tableName)); + } } else { throw e; } @@ -232,10 +244,15 @@ public boolean next() throws Exception { referToNewPartition(entry.getValue()); } catch (Exception e) { if (e instanceof ObTableNeedFetchAllException) { - this.asyncRequest.getObTableQueryRequest().getTableQuery() - .adjustStartKey(currentStartKey); - setExpectant(refreshPartition(this.asyncRequest.getObTableQueryRequest() - .getTableQuery(), tableName)); + TableEntry tableEntry = client.getOrRefreshTableEntry(tableName, false, false, false); + if (ObGlobal.obVsnMajor() >= 4 + && tableEntry.isPartitionTable() + && tableEntry.getPartitionInfo().getFirstPartDesc().getPartFuncType().isRangePart()) { + this.asyncRequest.getObTableQueryRequest().getTableQuery() + .adjustStartKey(currentStartKey); + setExpectant(refreshPartition(this.asyncRequest.getObTableQueryRequest() + .getTableQuery(), tableName)); + } it = expectant.entrySet().iterator(); retryTimes++; if (retryTimes > client.getTableEntryRefreshTryTimes()) { From e5762f4d2b9e385231b5b2449ce1a4c5e7d8fcc4 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 23 Oct 2024 20:09:40 +0800 Subject: [PATCH 21/23] add -4138 --- .../oceanbase/rpc/bolt/transport/ObTableRemoting.java | 5 ++--- .../com/alipay/oceanbase/rpc/property/Property.java | 2 +- .../java/com/alipay/oceanbase/rpc/table/ObTable.java | 10 +++++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java index 69f90bc9..37a708bf 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java @@ -31,8 +31,6 @@ import io.netty.buffer.ByteBuf; import org.slf4j.Logger; -import javax.xml.transform.Result; - import static com.alipay.oceanbase.rpc.protocol.packet.ObCompressType.INVALID_COMPRESSOR; import static com.alipay.oceanbase.rpc.protocol.packet.ObCompressType.NONE_COMPRESSOR; @@ -124,7 +122,7 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques ObRpcResultCode resultCode = new ObRpcResultCode(); resultCode.decode(buf); // If response indicates the request is routed to wrong server, we should refresh the routing meta. - if (!conn.getObTable().getReRouting() && response.getHeader().isRoutingWrong()) { + if (!conn.getObTable().isEnableRerouting() && response.getHeader().isRoutingWrong()) { String errMessage = TraceUtil.formatTraceMessage(conn, request, "routed to the wrong server: " + response.getMessage()); logger.warn(errMessage); @@ -197,6 +195,7 @@ private boolean needFetchAll(int errorCode, int pcode) { || errorCode == ResultCodes.OB_TABLET_NOT_EXIST.errorCode || errorCode == ResultCodes.OB_LS_NOT_EXIST.errorCode || errorCode == ResultCodes.OB_MAPPING_BETWEEN_TABLET_AND_LS_NOT_EXIST.errorCode + || errorCode == ResultCodes.OB_SNAPSHOT_DISCARDED.errorCode || (pcode == Pcodes.OB_TABLE_API_LS_EXECUTE && errorCode == ResultCodes.OB_NOT_MASTER.errorCode); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/property/Property.java b/src/main/java/com/alipay/oceanbase/rpc/property/Property.java index abdfbfde..8ee42ebb 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/property/Property.java +++ b/src/main/java/com/alipay/oceanbase/rpc/property/Property.java @@ -134,7 +134,7 @@ public enum Property { NETTY_BLOCKING_WAIT_INTERVAL("bolt.netty.blocking.wait.interval", 1, "netty写缓存满后等待时间"), // [ObTable][OTHERS] - SERVER_ENABLE_REROUTING("server.enable.rerouting", true, "开启server端的重定向回复功能"), + SERVER_ENABLE_REROUTING("server.enable.rerouting", false, "开启server端的重定向回复功能"), /* * other config diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java index fdbce217..551782dd 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java @@ -66,7 +66,7 @@ public class ObTable extends AbstractObTable implements Lifecycle { private volatile boolean initialized = false; private volatile boolean closed = false; - private boolean reRouting = true; // only used for init packet factory + private boolean enableRerouting = true; // only used for init packet factory private ReentrantLock statusLock = new ReentrantLock(); @@ -89,7 +89,7 @@ public void init() throws Exception { .configWriteBufferWaterMark(getNettyBufferLowWatermark(), getNettyBufferHighWatermark()).build(); connectionFactory.init(new ConnectionEventHandler(new GlobalSwitch())); // Only for monitoring connection status - realClient = new ObTableRemoting(new ObPacketFactory(reRouting)); + realClient = new ObTableRemoting(new ObPacketFactory(enableRerouting)); connectionPool = new ObTableConnectionPool(this, obTableConnectionPoolSize); connectionPool.init(); initialized = true; @@ -164,7 +164,7 @@ private void initProperties() { nettyBufferHighWatermark); nettyBlockingWaitInterval = parseToInt(NETTY_BLOCKING_WAIT_INTERVAL.getKey(), nettyBlockingWaitInterval); - reRouting = parseToBoolean(SERVER_ENABLE_REROUTING.getKey(), reRouting); + enableRerouting = parseToBoolean(SERVER_ENABLE_REROUTING.getKey(), enableRerouting); maxConnExpiredTime = parseToLong(MAX_CONN_EXPIRED_TIME.getKey(), maxConnExpiredTime); Object value = this.configs.get("runtime"); @@ -174,8 +174,8 @@ private void initProperties() { } } - public boolean getReRouting(){ - return reRouting; + public boolean isEnableRerouting(){ + return enableRerouting; } /* From 4c630eedc5f318459803c4ebaed2c18046a20c08 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Fri, 25 Oct 2024 11:51:45 +0800 Subject: [PATCH 22/23] fix review: add ut for byteutil --- .../table/ObTableClientLSBatchOpsImpl.java | 15 ++-- .../oceanbase/rpc/util/ByteUtilTest.java | 85 +++++++++++++++++++ 2 files changed, 92 insertions(+), 8 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/rpc/util/ByteUtilTest.java diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index 9f6ad172..2d8e7459 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -360,18 +360,17 @@ public Map>>> tabletOperations + Map>>> tabletOperations = lsOperationsMap.computeIfAbsent(lsId, k -> new HashMap<>()); - // if ls id not exists + // if ls id not exists - ObPair>> singleOperations = + ObPair>> singleOperations = tabletOperations.computeIfAbsent(tableObPair.getLeft(), k -> new ObPair<>(tableObPair.getRight(), new ArrayList<>())); - // if tablet id not exists - - singleOperations.getRight().add(new ObPair<>(i, operation)); - } + // if tablet id not exists + singleOperations.getRight().add(new ObPair<>(i, operation)); + } - return lsOperationsMap; + return lsOperationsMap; } public Map>>>> partitionPrepare() diff --git a/src/test/java/com/alipay/oceanbase/rpc/util/ByteUtilTest.java b/src/test/java/com/alipay/oceanbase/rpc/util/ByteUtilTest.java new file mode 100644 index 00000000..92dae7dc --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/util/ByteUtilTest.java @@ -0,0 +1,85 @@ +package com.alipay.oceanbase.rpc.util; + +import org.junit.Assert; +import org.junit.Test; + +import static com.alipay.oceanbase.rpc.util.ByteUtil.compareByteArrays; +import static org.junit.Assert.*; + + +public class ByteUtilTest { + @Test + public void testcompareByteArrays() { + { + byte[] array1 = {1, 2, 3}; + byte[] array2 = {1, 2, 3}; + Assert.assertEquals(0, compareByteArrays(array1, array2)); + } + { + byte[] array1 = {2, 2, 3}; + byte[] array2 = {1, 2, 3}; + Assert.assertTrue(compareByteArrays(array1, array2) > 0); + } + { + byte[] array1 = {1, 2, 3, 4}; + byte[] array2 = {1, 2, 3}; + assertTrue(compareByteArrays(array1, array2) > 0); + } + { + byte[] array1 = {}; + byte[] array2 = {}; + assertEquals(0, compareByteArrays(array1, array2)); + } + } + @Test + public void testincrementByteArray() { + { + byte[] input = {0x01, 0x02, 0x03}; + byte[] expected = {0x01, 0x02, 0x04}; + assertArrayEquals(expected, ByteUtil.incrementByteArray(input)); + } + { + byte[] input = {(byte) 0xFF, (byte) 0xFF}; + byte[] expected = {0x01, 0x00, 0x00}; + assertArrayEquals(expected, ByteUtil.incrementByteArray(input)); + } + { + byte[] input = {}; + byte[] expected = {0x01}; + assertArrayEquals(expected, ByteUtil.incrementByteArray(input)); + } + { + byte[] expected = {0x01}; + assertArrayEquals(expected, ByteUtil.incrementByteArray(null)); + } + } + + @Test + public void testdecrementByteArray() { + { + byte[] input = {0x01}; + byte[] expected = {0x00}; + assertArrayEquals(expected, ByteUtil.decrementByteArray(input)); + } + { + byte[] input = {0x01, 0x00}; + byte[] expected = {0x00, (byte) 0xFF}; + assertArrayEquals(expected, ByteUtil.decrementByteArray(input)); + } + { + byte[] input = {0x02, 0x00}; + byte[] expected = {0x01, (byte) 0xFF}; + assertArrayEquals(expected, ByteUtil.decrementByteArray(input)); + } + { + byte[] input = {0x01, 0x00, 0x00}; + byte[] expected = {0x00, (byte) 0xFF, (byte) 0xFF}; + assertArrayEquals(expected, ByteUtil.decrementByteArray(input)); + } + { + byte[] input = {(byte) 0xFF, (byte) 0xFF}; + byte[] expected = {(byte) 0xFF, (byte) 0xFE}; + assertArrayEquals(expected, ByteUtil.decrementByteArray(input)); + } + } +} From 64926a44c9b30681fec4aadf8575a393d0b8a1df Mon Sep 17 00:00:00 2001 From: maochongxin Date: Fri, 25 Oct 2024 18:45:03 +0800 Subject: [PATCH 23/23] fix lsop refresh location --- .../oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java index 2d8e7459..6459398c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -21,6 +21,7 @@ import com.alipay.oceanbase.rpc.checkandmutate.CheckAndInsUp; import com.alipay.oceanbase.rpc.exception.*; import com.alipay.oceanbase.rpc.location.model.ObServerRoute; +import com.alipay.oceanbase.rpc.location.model.TableEntry; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.mutation.*; import com.alipay.oceanbase.rpc.mutation.result.MutationResult; @@ -473,6 +474,9 @@ public void partitionExecute(ObTableSingleOpResult[] results, if (failedServerList != null) { route.setBlackList(failedServerList); } + TableEntry entry = obTableClient.getOrRefreshTableEntry(tableName, false, + false, false); + obTableClient.refreshTableLocationByTabletId(entry, tableName, obTableClient.getTabletIdByPartId(entry, originPartId)); subObTable = obTableClient.getTableWithPartId(tableName, originPartId, needRefreshTableEntry, obTableClient.isTableEntryRefreshIntervalWait(), false, route). getRight().getObTable();