From 71579fd0b74decb07aa96adf6f1c392cb781f5b9 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 20 Aug 2024 14:57:42 +0800 Subject: [PATCH 1/5] merged all commits into one --- .github/workflows/ci.yaml | 60 + .gitignore | 1 + .secignore | 35 + README.md | 43 +- core-0.12.0.jar | Bin 114772 -> 0 bytes example/simple-kv-demo/README.md | 1 + example/simple-kv-demo/pom.xml | 26 + .../java/com/oceanbase/example/KVClient.java | 228 ++ .../com/oceanbase/example/LoggerFactory.java | 44 + .../com/oceanbase/example/SimpleKVDemo.java | 106 + example/simple-mutation/README.md | 4 + example/simple-mutation/pom.xml | 26 + .../com/oceanbase/example/MutationUtil.java | 71 + .../com/oceanbase/example/SimpleMutation.java | 199 ++ example/simple-table-demo/README.md | 0 example/simple-table-demo/pom.xml | 26 + .../com/oceanbase/example/LoggerFactory.java | 44 + .../example/ObTableDirectLoadExample.java | 281 ++ .../oceanbase/example/SimpleTableDemo.java | 124 + .../com/oceanbase/example/TableClient.java | 296 ++ pom.xml | 131 +- .../oceanbase/rpc/ObClusterTableBatchOps.java | 75 +- .../oceanbase/rpc/ObClusterTableQuery.java | 70 +- .../com/alipay/oceanbase/rpc/ObGlobal.java | 95 + .../alipay/oceanbase/rpc/ObTableClient.java | 2602 +++++++++++++++-- .../oceanbase/rpc/batch/QueryByBatch.java | 259 -- .../rpc/batch/QueryByBatchResultSet.java | 120 - .../rpc/bolt/protocol/ObTablePacket.java | 55 +- .../rpc/bolt/protocol/ObTablePacketCode.java | 70 +- .../bolt/protocol/ObTablePacketDecoder.java | 2 +- .../bolt/protocol/ObTablePacketEncoder.java | 2 +- .../rpc/bolt/protocol/ObTableProtocol.java | 14 +- .../rpc/bolt/transport/ObClientFuture.java | 72 +- .../transport/ObConfigurableInstance.java | 2 +- .../bolt/transport/ObConnectionFactory.java | 18 +- .../rpc/bolt/transport/ObPacketCodec.java | 4 +- .../rpc/bolt/transport/ObPacketFactory.java | 34 +- .../rpc/bolt/transport/ObTableConnection.java | 164 +- .../bolt/transport/ObTablePacketHandler.java | 12 +- .../transport/ObTablePacketProcessor.java | 7 +- .../rpc/bolt/transport/ObTableRemoting.java | 101 +- .../rpc/checkandmutate/CheckAndInsUp.java | 101 + .../oceanbase/rpc/constant/Constants.java | 1 + .../rpc/exception/ExceptionUtil.java | 132 +- .../FeatureNotSupportedException.java | 14 +- .../GenerateColumnParseException.java | 14 +- .../rpc/exception/ObShardTableException.java | 14 +- .../rpc/exception/ObTableAuthException.java | 12 +- .../rpc/exception/ObTableCloseException.java | 12 +- .../ObTableConnectionStatusException.java | 12 +- .../ObTableConnectionUnWritableException.java | 12 +- .../ObTableDuplicateKeyException.java | 12 +- .../ObTableEntryRefreshException.java | 14 +- .../rpc/exception/ObTableException.java | 16 +- .../rpc/exception/ObTableGetException.java | 12 +- .../ObTableGlobalIndexRouteException.java | 69 + .../rpc/exception/ObTableLoginException.java | 12 +- .../ObTableMasterChangeException.java | 14 +- .../ObTableNeedFetchAllException.java | 68 + .../exception/ObTableNoMasterException.java | 12 +- .../ObTableNoReadableReplicaException.java | 14 +- .../exception/ObTableNotExistException.java | 12 +- .../ObTablePartitionChangeException.java | 14 +- .../ObTablePartitionConsistentException.java | 14 +- .../ObTablePartitionInfoRefreshException.java | 12 +- ...ablePartitionLocationRefreshException.java | 14 +- .../ObTablePartitionNoMasterException.java | 14 +- .../ObTablePartitionNotExistException.java | 12 +- .../ObTableReplicaNotReadableException.java | 14 +- .../ObTableRetryExhaustedException.java | 12 +- .../ObTableRoutingWrongException.java | 14 +- .../ObTableServerCacheExpiredException.java | 14 +- .../ObTableServerConnectException.java | 14 +- .../exception/ObTableServerDownException.java | 14 +- .../ObTableServerStatusChangeException.java | 14 +- .../ObTableServerTimeoutException.java | 12 +- .../ObTableTenantNotInServerException.java | 69 + .../rpc/exception/ObTableTimeoutExcetion.java | 12 +- .../ObTableTransactionRpcTimeout.java | 14 +- .../exception/ObTableTransportException.java | 14 +- .../exception/ObTableUnexpectedException.java | 12 +- .../ObTableUnitMigrateException.java | 14 +- .../oceanbase/rpc/filter/ObCompareOp.java | 98 + .../oceanbase/rpc/filter/ObTableFilter.java | 31 + .../rpc/filter/ObTableFilterFactory.java | 54 + .../rpc/filter/ObTableFilterList.java | 134 + .../oceanbase/rpc/filter/ObTableInFilter.java | 91 + .../rpc/filter/ObTableNotInFilter.java | 84 + .../rpc/filter/ObTableValueFilter.java | 96 + .../oceanbase/rpc/location/LocationUtil.java | 1211 ++++++-- .../location/SecureIdentityLoginModule.java | 4 +- .../rpc/location/model/ObIndexInfo.java | 73 + .../rpc/location/model/ObReadConsistency.java | 14 +- .../rpc/location/model/ObReplicaType.java | 10 +- .../rpc/location/model/ObRoutePolicy.java | 4 +- .../rpc/location/model/ObServerAddr.java | 32 +- .../rpc/location/model/ObServerInfo.java | 12 +- .../rpc/location/model/ObServerLdcItem.java | 12 +- .../location/model/ObServerLdcLocation.java | 18 +- .../rpc/location/model/ObServerRole.java | 4 +- .../rpc/location/model/ObServerRoute.java | 18 +- .../rpc/location/model/ObUserAuth.java | 16 +- .../rpc/location/model/OcpModel.java | 18 +- .../rpc/location/model/OcpResponse.java | 20 +- .../rpc/location/model/OcpResponseData.java | 20 +- .../location/model/OcpResponseDataIDC.java | 10 +- .../rpc/location/model/OcpResponseDataRs.java | 14 +- .../rpc/location/model/ReplicaLocation.java | 28 +- .../rpc/location/model/ServerRoster.java | 28 +- .../rpc/location/model/TableEntry.java | 84 +- .../rpc/location/model/TableEntryKey.java | 36 +- .../rpc/location/model/TableLocation.java | 6 +- .../model/partition/ObComparableKV.java | 6 +- .../model/partition/ObHashPartDesc.java | 103 +- .../model/partition/ObKeyPartDesc.java | 163 +- .../model/partition/ObListPartDesc.java | 62 +- .../rpc/location/model/partition/ObPair.java | 6 +- .../model/partition/ObPartConstants.java | 4 +- .../location/model/partition/ObPartDesc.java | 163 +- .../model/partition/ObPartFuncType.java | 28 +- .../model/partition/ObPartIdCalculator.java | 49 +- .../model/partition/ObPartitionEntry.java | 38 +- .../model/partition/ObPartitionInfo.java | 51 +- .../model/partition/ObPartitionKey.java | 29 +- .../model/partition/ObPartitionLevel.java | 6 +- .../model/partition/ObPartitionLocation.java | 36 +- .../model/partition/ObRangePartDesc.java | 202 +- .../alipay/oceanbase/rpc/mutation/Append.java | 147 + .../rpc/mutation/BatchOperation.java | 320 ++ .../oceanbase/rpc/mutation/ColumnValue.java | 51 + .../alipay/oceanbase/rpc/mutation/Delete.java | 75 + .../oceanbase/rpc/mutation/Increment.java | 146 + .../alipay/oceanbase/rpc/mutation/Insert.java | 171 ++ .../rpc/mutation/InsertOrUpdate.java | 148 + .../oceanbase/rpc/mutation/Mutation.java | 493 ++++ .../rpc/mutation/MutationFactory.java | 73 + .../alipay/oceanbase/rpc/mutation/Put.java | 169 ++ .../oceanbase/rpc/mutation/Replace.java | 136 + .../alipay/oceanbase/rpc/mutation/Row.java | 141 + .../alipay/oceanbase/rpc/mutation/Update.java | 141 + .../mutation/result/BatchOperationResult.java | 155 + .../rpc/mutation/result/MutationResult.java | 90 + .../rpc/mutation/result/OperationResult.java | 58 + .../rpc/property/AbstractPropertyAware.java | 29 +- .../oceanbase/rpc/property/Property.java | 90 +- .../rpc/protocol/packet/ObCompressType.java | 4 +- .../rpc/protocol/packet/ObRpcCostTime.java | 34 +- .../rpc/protocol/packet/ObRpcPacket.java | 12 +- .../protocol/packet/ObRpcPacketHeader.java | 189 +- .../rpc/protocol/payload/AbstractPayload.java | 47 +- .../rpc/protocol/payload/Constants.java | 12 +- .../rpc/protocol/payload/ObPayload.java | 36 +- .../rpc/protocol/payload/ObRpcResultCode.java | 30 +- .../payload/ObRpcResultWarningMsg.java | 30 +- .../rpc/protocol/payload/ObSimplePayload.java | 7 +- .../rpc/protocol/payload/ObUnisVersion.java | 7 - .../rpc/protocol/payload/Pcodes.java | 17 +- .../rpc/protocol/payload/ResultCodes.java | 20 +- .../rpc/protocol/payload/impl/ObAddr.java | 150 + .../payload/impl/ObCollationLevel.java | 6 +- .../payload/impl/ObCollationType.java | 6 +- .../rpc/protocol/payload/impl/ObColumn.java | 16 +- .../rpc/protocol/payload/impl/ObObj.java | 77 +- .../rpc/protocol/payload/impl/ObObjMeta.java | 38 +- .../rpc/protocol/payload/impl/ObObjType.java | 499 ++-- .../rpc/protocol/payload/impl/ObRowKey.java | 25 +- .../protocol/payload/impl/ObTableObjType.java | 360 +++ .../payload/impl/ObTableSerialUtil.java | 153 + .../impl/column/ObGeneratedColumn.java | 4 +- .../column/ObGeneratedColumnReferFunc.java | 12 +- .../column/ObGeneratedColumnSubStrFunc.java | 20 +- .../payload/impl/column/ObSimpleColumn.java | 4 +- .../impl/direct_load/ObLoadDupActionType.java | 52 + .../ObTableDirectLoadAbortArg.java | 92 + .../ObTableDirectLoadBeginArg.java | 160 + .../ObTableDirectLoadBeginRes.java | 148 + .../ObTableDirectLoadCommitArg.java | 92 + .../ObTableDirectLoadGetStatusArg.java | 92 + .../ObTableDirectLoadGetStatusRes.java | 93 + .../ObTableDirectLoadHeartBeatArg.java | 92 + .../ObTableDirectLoadHeartBeatRes.java | 93 + .../ObTableDirectLoadInsertArg.java | 109 + .../ObTableDirectLoadOperationType.java | 52 + .../direct_load/ObTableDirectLoadRequest.java | 195 ++ .../direct_load/ObTableDirectLoadResult.java | 169 ++ .../direct_load/ObTableLoadClientStatus.java | 52 + .../impl/execute/AbstractObTableEntity.java | 8 +- .../payload/impl/execute/ObITableEntity.java | 4 +- .../payload/impl/execute/ObIndexType.java | 74 + .../ObTableAbstractOperationRequest.java | 92 +- .../impl/execute/ObTableBatchOperation.java | 100 +- .../execute/ObTableBatchOperationRequest.java | 48 +- .../execute/ObTableBatchOperationResult.java | 17 +- .../impl/execute/ObTableConsistencyLevel.java | 6 +- .../payload/impl/execute/ObTableEntity.java | 18 +- .../impl/execute/ObTableEntityType.java | 6 +- .../payload/impl/execute/ObTableLSOpFlag.java | 78 + .../impl/execute/ObTableLSOpRequest.java | 156 + .../impl/execute/ObTableLSOpResult.java | 148 + .../impl/execute/ObTableLSOperation.java | 344 +++ .../impl/execute/ObTableOperation.java | 42 +- .../impl/execute/ObTableOperationRequest.java | 28 +- .../impl/execute/ObTableOperationResult.java | 38 +- .../impl/execute/ObTableOperationType.java | 38 +- .../impl/execute/ObTableOptionFlag.java | 104 + .../execute/ObTableQueryAndMutateFlag.java | 52 + .../payload/impl/execute/ObTableResult.java | 28 +- .../payload/impl/execute/ObTableSingleOp.java | 170 ++ .../impl/execute/ObTableSingleOpEntity.java | 369 +++ .../impl/execute/ObTableSingleOpFlag.java | 43 + .../impl/execute/ObTableSingleOpQuery.java | 232 ++ .../impl/execute/ObTableSingleOpResult.java | 200 ++ .../impl/execute/ObTableSingleOpType.java | 74 + .../impl/execute/ObTableStreamRequest.java | 18 +- .../payload/impl/execute/ObTableTabletOp.java | 193 ++ .../impl/execute/ObTableTabletOpFlag.java | 70 + .../impl/execute/ObTableTabletOpResult.java | 130 + .../aggregation/ObTableAggregation.java | 119 + .../aggregation/ObTableAggregationResult.java | 60 + .../aggregation/ObTableAggregationSingle.java | 63 + .../aggregation/ObTableAggregationType.java | 57 + .../execute/mutate/ObTableQueryAndMutate.java | 57 +- .../mutate/ObTableQueryAndMutateRequest.java | 47 +- .../mutate/ObTableQueryAndMutateResult.java | 16 +- .../query/AbstractQueryStreamResult.java | 358 ++- .../impl/execute/query/ObBorderFlag.java | 30 +- .../impl/execute/query/ObHTableFilter.java | 42 +- .../impl/execute/query/ObNewRange.java | 85 +- .../impl/execute/query/ObScanOrder.java | 6 +- .../impl/execute/query/ObTableQuery.java | 150 +- .../execute/query/ObTableQueryRequest.java | 28 +- .../execute/query/ObTableQueryResult.java | 38 +- .../syncquery/ObQueryOperationType.java | 50 + .../syncquery/ObTableQueryAsyncRequest.java | 127 + .../syncquery/ObTableQueryAsyncResult.java | 108 + .../impl/login/ObTableLoginRequest.java | 70 +- .../impl/login/ObTableLoginResult.java | 42 +- .../payload/impl/parser/LayoutCharacters.java | 12 +- .../parser/ObGeneratedColumnCharType.java | 10 +- .../ObGeneratedColumnExpressParser.java | 6 +- .../parser/ObGeneratedColumnFuncName.java | 2 +- .../impl/parser/ObGeneratedColumnLexer.java | 18 +- .../ObTableClientQueryAsyncStreamResult.java | 277 ++ .../ObTableClientQueryStreamResult.java | 118 +- .../rpc/stream/ObTableQueryStreamResult.java | 36 - .../oceanbase/rpc/stream/QueryResultSet.java | 17 +- .../oceanbase/rpc/table/AbstractObTable.java | 34 +- .../rpc/table/AbstractObTableClient.java | 137 +- .../oceanbase/rpc/table/AbstractTable.java | 10 +- .../rpc/table/AbstractTableBatchOps.java | 34 +- .../rpc/table/AbstractTableQuery.java | 64 +- .../rpc/table/AbstractTableQueryImpl.java | 46 +- .../oceanbase/rpc/table/ConcurrentTask.java | 2 +- .../rpc/table/ConcurrentTaskExecutor.java | 14 +- .../rpc/table/ObDirectLoadBucket.java | 109 + .../rpc/table/ObDirectLoadObjRow.java | 124 + .../rpc/table/ObDirectLoadParameter.java | 93 + .../alipay/oceanbase/rpc/table/ObTable.java | 292 +- .../rpc/table/ObTableBatchOpsImpl.java | 88 +- .../rpc/table/ObTableClientBatchOpsImpl.java | 339 ++- .../table/ObTableClientLSBatchOpsImpl.java | 703 +++++ .../rpc/table/ObTableClientQueryImpl.java | 256 +- .../rpc/table/ObTableDirectLoad.java | 368 +++ .../oceanbase/rpc/table/ObTableParam.java | 133 + .../oceanbase/rpc/table/ObTableQueryImpl.java | 121 - .../alipay/oceanbase/rpc/table/api/Table.java | 26 +- .../rpc/table/api/TableBatchOps.java | 10 + .../oceanbase/rpc/table/api/TableQuery.java | 66 +- .../rpc/threadlocal/ThreadLocalMap.java | 22 +- .../oceanbase/rpc/util/AsyncExecutor.java | 19 +- .../com/alipay/oceanbase/rpc/util/Base64.java | 46 +- .../com/alipay/oceanbase/rpc/util/CRC64.java | 18 +- .../oceanbase/rpc/util/MonitorUtil.java | 316 ++ .../oceanbase/rpc/util/NamePattern.java | 13 +- .../rpc/util/NamedThreadFactory.java | 7 +- .../alipay/oceanbase/rpc/util/ObByteBuf.java | 92 + .../oceanbase/rpc/util/ObBytesString.java | 42 +- .../oceanbase/rpc/util/ObHashUtils.java | 94 +- .../oceanbase/rpc/util/ObPureCrc32C.java | 10 +- .../alipay/oceanbase/rpc/util/ObVString.java | 23 +- .../alipay/oceanbase/rpc/util/Security.java | 141 +- .../oceanbase/rpc/util/Serialization.java | 411 ++- .../alipay/oceanbase/rpc/util/StringUtil.java | 12 +- .../rpc/util/TableClientLoggerFactory.java | 62 +- .../alipay/oceanbase/rpc/util/TimeUtils.java | 16 +- .../alipay/oceanbase/rpc/util/TraceUtil.java | 18 +- .../oceanbase/rpc/util/WrappedLogger.java | 374 +++ .../alipay/oceanbase/rpc/util/YcsbBench.java | 148 - .../oceanbase/rpc/util/YcsbBenchClient.java | 208 -- .../alipay/oceanbase/rpc/util/ZoneUtil.java | 2 +- .../oceanbase/rpc/util/hash/MurmurHash.java | 55 + .../rpc/util/hash/ObHashSortBin.java | 2 +- .../rpc/util/hash/ObHashSortUtf8mb4.java | 67 +- .../oceanbase/rpc/util/hash/ObMbContext.java | 3 + .../rpc/util/hash/ObUnicaseInfo.java | 2 +- .../rpc/util/hash/ObUnicaseInfoChar.java | 2 +- .../log-codes.properties | 2 +- .../log/log4j/log-conf.xml | 60 +- .../log/log4j2/log-conf.xml | 51 + .../log/logback/log-conf.xml | 111 +- src/test/java/ci.sql | 128 - .../rpc/ObAtomicBatchOperationTest.java | 286 +- .../oceanbase/rpc/ObTableAggregationTest.java | 633 ++++ .../oceanbase/rpc/ObTableAuditTest.java | 608 ++++ .../rpc/ObTableCheckAndInsUpTest.java | 537 ++++ .../rpc/ObTableClientAutoIncTest.java | 567 ++++ .../rpc/ObTableClientCheckAndInsertTest.java | 178 ++ .../oceanbase/rpc/ObTableClientITCase.java | 68 + .../oceanbase/rpc/ObTableClientInitTest.java | 1 - .../rpc/ObTableClientPartitionHashTest.java | 485 ++- .../rpc/ObTableClientPartitionKeyTest.java | 1373 +++++++-- .../rpc/ObTableClientPartitionRangeTest.java | 1067 +++++-- .../oceanbase/rpc/ObTableClientTest.java | 2274 +++++++++++++- .../oceanbase/rpc/ObTableConnectionTest.java | 173 ++ .../oceanbase/rpc/ObTableErrMsgTest.java | 239 ++ .../oceanbase/rpc/ObTableGlobalIndexTest.java | 583 ++++ .../rpc/ObTableHotkeyThrottleTest.java | 94 + .../rpc/ObTableIndexWithCalcColumn.java | 633 ++++ .../oceanbase/rpc/ObTableLsBatchTest.java | 751 +++++ .../alipay/oceanbase/rpc/ObTableModeTest.java | 316 ++ .../alipay/oceanbase/rpc/ObTableP99Test.java | 385 +++ .../alipay/oceanbase/rpc/ObTablePutTest.java | 156 + .../oceanbase/rpc/ObTableSQLAuditTest.java | 307 ++ .../alipay/oceanbase/rpc/ObTableTTLTest.java | 488 ++++ .../rpc/ObWeakReadConsistencyTest.java | 55 +- .../rpc/bolt/ObTableClientTestBase.java | 703 +++-- .../oceanbase/rpc/bolt/ObTableTest.java | 25 +- .../rpc/containerBase/ContainerTestBase.java | 96 + .../rpc/exception/ExceptionUtilTest.java | 11 +- .../rpc/hbase/ObHTableOperationRequest.java | 68 +- .../oceanbase/rpc/hbase/ObHTableTest.java | 180 +- .../model/partition/ObHashPartDescTest.java | 682 ++++- .../model/partition/ObKeyPartDescTest.java | 418 ++- .../model/partition/ObRangePartDescTest.java | 429 ++- .../protocol/codec/ObRpcPacketHeaderTest.java | 2 +- .../ObTableBatchOperationResultTest.java | 2 +- .../execute/ObTableOperationResultTest.java | 2 +- .../query/ObTableQueryAsyncPayloadTest.java | 151 + .../rpc/table/ObTableConnectionTest.java | 94 +- .../alipay/oceanbase/rpc/temp/Crc64Util.java | 1 - .../oceanbase/rpc/util/ObHashUtilTest.java | 28 +- .../rpc/util/ObTableClientTestUtil.java | 112 +- .../rpc/util/ObTableHotkeyThrottleUtil.java | 420 +++ .../oceanbase/rpc/util/YcsbBenchTest.java | 118 - src/test/resources/ci.sql | 600 ++++ src/test/resources/log4j.properties | 6 - src/test/resources/logback-test.xml | 12 + 347 files changed, 37011 insertions(+), 5620 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 .secignore delete mode 100644 core-0.12.0.jar create mode 100644 example/simple-kv-demo/README.md create mode 100644 example/simple-kv-demo/pom.xml create mode 100644 example/simple-kv-demo/src/main/java/com/oceanbase/example/KVClient.java create mode 100644 example/simple-kv-demo/src/main/java/com/oceanbase/example/LoggerFactory.java create mode 100644 example/simple-kv-demo/src/main/java/com/oceanbase/example/SimpleKVDemo.java create mode 100644 example/simple-mutation/README.md create mode 100644 example/simple-mutation/pom.xml create mode 100644 example/simple-mutation/src/main/java/com/oceanbase/example/MutationUtil.java create mode 100644 example/simple-mutation/src/main/java/com/oceanbase/example/SimpleMutation.java create mode 100644 example/simple-table-demo/README.md create mode 100644 example/simple-table-demo/pom.xml create mode 100644 example/simple-table-demo/src/main/java/com/oceanbase/example/LoggerFactory.java create mode 100644 example/simple-table-demo/src/main/java/com/oceanbase/example/ObTableDirectLoadExample.java create mode 100644 example/simple-table-demo/src/main/java/com/oceanbase/example/SimpleTableDemo.java create mode 100644 example/simple-table-demo/src/main/java/com/oceanbase/example/TableClient.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/ObGlobal.java delete mode 100644 src/main/java/com/alipay/oceanbase/rpc/batch/QueryByBatch.java delete mode 100644 src/main/java/com/alipay/oceanbase/rpc/batch/QueryByBatchResultSet.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/checkandmutate/CheckAndInsUp.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/exception/ObTableGlobalIndexRouteException.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNeedFetchAllException.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTenantNotInServerException.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/filter/ObCompareOp.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilter.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilterFactory.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilterList.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/filter/ObTableInFilter.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/filter/ObTableNotInFilter.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/filter/ObTableValueFilter.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/location/model/ObIndexInfo.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/Append.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/ColumnValue.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/Delete.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/Increment.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/Insert.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/InsertOrUpdate.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/Mutation.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/MutationFactory.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/Replace.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/Row.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/Update.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/result/BatchOperationResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/result/MutationResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/mutation/result/OperationResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObAddr.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableSerialUtil.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObLoadDupActionType.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadAbortArg.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadBeginArg.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadBeginRes.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadCommitArg.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadGetStatusArg.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadGetStatusRes.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadHeartBeatArg.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadHeartBeatRes.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadInsertArg.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadOperationType.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadRequest.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableLoadClientStatus.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObIndexType.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpFlag.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpRequest.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOperation.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableQueryAndMutateFlag.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOp.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpEntity.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpFlag.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpQuery.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpType.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOp.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpFlag.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregation.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationSingle.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationType.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObQueryOperationType.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObTableQueryAsyncRequest.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObTableQueryAsyncResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java delete mode 100644 src/main/java/com/alipay/oceanbase/rpc/stream/ObTableQueryStreamResult.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadBucket.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadObjRow.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadParameter.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/table/ObTableDirectLoad.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/table/ObTableParam.java delete mode 100644 src/main/java/com/alipay/oceanbase/rpc/table/ObTableQueryImpl.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/util/MonitorUtil.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/util/ObByteBuf.java create mode 100644 src/main/java/com/alipay/oceanbase/rpc/util/WrappedLogger.java delete mode 100644 src/main/java/com/alipay/oceanbase/rpc/util/YcsbBench.java delete mode 100644 src/main/java/com/alipay/oceanbase/rpc/util/YcsbBenchClient.java delete mode 100644 src/test/java/ci.sql create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableAggregationTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableAuditTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableCheckAndInsUpTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableClientAutoIncTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableClientCheckAndInsertTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableClientITCase.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableConnectionTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableErrMsgTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableGlobalIndexTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableHotkeyThrottleTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableIndexWithCalcColumn.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableModeTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableP99Test.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableSQLAuditTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/ObTableTTLTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/containerBase/ContainerTestBase.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryAsyncPayloadTest.java create mode 100644 src/test/java/com/alipay/oceanbase/rpc/util/ObTableHotkeyThrottleUtil.java delete mode 100644 src/test/java/com/alipay/oceanbase/rpc/util/YcsbBenchTest.java create mode 100644 src/test/resources/ci.sql delete mode 100644 src/test/resources/log4j.properties create mode 100644 src/test/resources/logback-test.xml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..6b6eb42e --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,60 @@ +#- +# #%L +# OBKV Table Client Framework +# %% +# Copyright (C) 2023 OceanBase +# %% +# OBKV Table Client Framework is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# #L% + +name: CI + +on: + workflow_dispatch: + push: + branches: + - master + paths-ignore: + - 'example/**' + - '**.md' + - '.*' + pull_request: + paths-ignore: + - 'example/**' + - '**.md' + - '.*' + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'zulu' + cache: 'maven' + - name: Maven Build + run: mvn -B clean verify -Dgpg.skip -DskipTests=true + + test-obkv-table-client-java: + name: Test (obkv-table-client-java) + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'zulu' + cache: 'maven' + - name: Maven Test + run: mvn verify -Dgpg.skip -Dtest=ObTableClientITCase diff --git a/.gitignore b/.gitignore index abd27790..5db55ead 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ *.rar *.tar *.zip +*.jar # Logs and databases # ###################### diff --git a/.secignore b/.secignore new file mode 100644 index 00000000..97f9d019 --- /dev/null +++ b/.secignore @@ -0,0 +1,35 @@ +########################################################## +# OBFlow Secret Scan Ignore List # +########################################################## +# Above the segmentation lines there are suspected privacy information # +# Please use the file name as the first line and the igored information # +# should be started with tab. # +# Under the segmentation lines there are the folders which you need to ignore # +########################################################## +** + http://license.coscl.org.cn/MulanPSL2 + https://open.oceanbase.com/ + https://open.oceanbase.com + https://github.com/oceanbase/oceanbase + https://github.com/oceanbase/oceanbase/issues + https://open.oceanbase.com + https://www.contributor-covenant.org/* + http://reveng.sourceforge.net/crc-catalogue/17plus.htm + http://murmurhash.googlepages.com/ +**/*.cpp + http://www.gnu.org/licenses/ +-------------------------------------------------------- +# Should use GLOB wildcard to config and analysis the ignored folder +# The root patch should start with '/' +/README.md +/.git/hooks/fsmonitor-watchman.sample +/pom.xml +/example/simple-table-demo/pom.xml +/example/simple-kv-demo/pom.xml +/CONTRIBUTING.md +/src/test/java/com/alipay/oceanbase/rpc/location/model/ObUserAuthTest.java +/src/main/java/com/alipay/oceanbase/rpc/util/Base64.java +/.oceanbase-common/** +/src/main/resources/oceanbase-table-client/log/log4j/** +-------------------------------------------------------- +# Config the ignored fold to escape the Chinese scan by GLOB wildcard \ No newline at end of file diff --git a/README.md b/README.md index a141d787..113ea05e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # OBKV Table Client OBKV Table Client is Java Library that can be used to access table data from [OceanBase](https://github.com/oceanbase/oceanbase) storage layer. Its access method is different from JDBC, it skips the SQL parsing layer, so it has significant performance advantage. -## Quick start +## Quick Start Create table in the OceanBase database: @@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS `test_varchar_table` ( `c1` varchar(20) NOT NULL, `c2` varchar(20) DEFAULT NULL, PRIMARY KEY (`c1`) -); +) PARTITION BY KEY(`c1`) PARTITIONS 10; ``` Import the dependency for your maven project: @@ -18,7 +18,7 @@ Import the dependency for your maven project: com.oceanbase obkv-table-client - 0.1.0 + 1.2.12 ``` @@ -30,19 +30,22 @@ The code demo: obTableClient.setParamURL("param_url"); obTableClient.setPassword("password"); obTableClient.setSysUserName("sys_user_name"); - obTableClient.setEncSysPassword("sys_sys_password"); - client.init(); - + obTableClient.setSysPassword("sys_user_passwd"); + obTableClient.init(); + + // set primary key for partition table + obTableClient.addRowKeyElement("test_varchar_table", new String[]{"c1"}); + // 2. single execute // return affectedRows - client.insert("test_varchar_table", "foo", new String[] { "c2" }, new String[] { "bar" }); + obTableClient.insert("test_varchar_table", "foo", new String[] { "c2" }, new String[] { "bar" }); // return Map - client.get("test_varchar_table", "foo", new String[] { "c2" }); + obTableClient.get("test_varchar_table", "foo", new String[] { "c2" }); // return affectedRows - client.delete("test_varchar_table", "foo"); + obTableClient.delete("test_varchar_table", "foo"); // 3. batch execute - TableBatchOps batchOps = client.batch("test_varchar_table"); + TableBatchOps batchOps = obTableClient.batch("test_varchar_table"); batchOps.insert("foo", new String[] { "c2" }, new String[] { "bar" }); batchOps.get("foo", new String[] { "c2" }); batchOps.delete("foo"); @@ -51,12 +54,18 @@ The code demo: // the results include 3 item: 1. affectedRows; 2. Map; 3. affectedRows. ``` **NOTE:** -param_url is generated by ConfigServer (link TODO). +1. param_url is generated by [ConfigServer](https://ask.oceanbase.com/t/topic/35601923). +2. More example [Demo](https://github.com/oceanbase/obkv-table-client-java/tree/master/example) +4. full_user_name: the user for accessing obkv, which format is `user_name@tenant_name#cluster_name` +5. sys_user_name: `root@sys` or `proxy@sys`, which have privileges to access routing system view + +## Release Notes +Latest release notes could be found in [Release notes](https://github.com/oceanbase/obkv-table-client-java/wiki#release-notes) ## Documentation -- English [link TODO] -- Simplified Chinese (简体中文) [link TODO] +- English [Coming soon] +- [Simplified Chinese (简体中文)](https://github.com/oceanbase/obkv-table-client-java/wiki/OBKV-Java%E5%AE%A2%E6%88%B7%E7%AB%AF-%E8%AF%B4%E6%98%8E%E6%96%87%E6%A1%A3) ## Licencing @@ -66,14 +75,14 @@ OBKV Table Client is under [MulanPSL - 2.0](http://license.coscl.org.cn/MulanPSL Contributions are warmly welcomed and greatly appreciated. Here are a few ways you can contribute: -- Raise us an issue [link TODO]. +- Raise us an [Issue](https://github.com/oceanbase/obkv-table-client-java/issues) - Submit Pull Requests. For details, see [How to contribute](CONTRIBUTING.md). ## Support In case you have any problems when using OceanBase Database, welcome reach out for help: -- GitHub Issue [link TODO] -- Official forum [link TODO] -- Knowledge base [link TODO] +- GitHub Issue [GitHub Issue](https://github.com/oceanbase/obkv-table-client-java/issues) +- Official forum [Official website](https://open.oceanbase.com) +- Knowledge base [Coming soon] diff --git a/core-0.12.0.jar b/core-0.12.0.jar deleted file mode 100644 index 36512c653535ffa384d710c6ad661a5fd6fc5940..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 114772 zcmagE19WClwlx|X72CFL+qP{d73+&_S8Ut1ZL4BCmAvZP_x|ti*FEk#<6w_*<{EqN zv)0C(XDP~ng24d&<4Ev{min*4f8U`0KIOzzh3KT@#TgX;8wLdw{};B|I@V>WE(Pftx)xiGdYGL0JMH89HvlJBwc+_f3D*=tvMn z5$KC!-a$t%n=`%6$>loT^0w&d*%biBHPHx*%mj4A;#zaVtS!|Q8)XUcrSwG4V&2D` z@C7^La|Ri;)A%|i!Oqb)M$SepCEH?qoIw3U)gFAm)AXM1>YU*WJ2oh7 zT@hSk>WAf9fAPu=F`dgK-fGX^lV7e@rpM#OhZF8sNOE9Bakmd902}CrIQcDcV_Dso3yYB8H`fatV%vkw4h8 zbQZ2_q&38bW|3tybaEOMjg<|GQ`Fj-)*Wbj7}7n1|6`oW`+=sxdA#3~!5s&s|8Qee ztyvZGN@G``G(eq-etscBZ>)s&>;H_5fBJ2hF2c7T76=Fd2MFjt#M*!R?VnzoPWSdf zT}B)A=lDn;Hzyni2}FwrLXsqmhztS*lqt6i#yok5MzQOs^a4(6QFN66&zejnNZxq1Oh? z(Lvd&GAu{kb2vc1bvfX1$H(=Cj@DYyHXXXXR(S9%EZ-?IoI_9Qn3lYy6XM&ZqkpY* z>kIzU!3e5R)~2v?qj492`W|@6TXsNHerJT?7yifjNWytp;^6 zJG*;MQV}Xj4$H)xQ-c57VHt8UvREnk>U4JO zm4G8Se+E#X3n#)6+@pDSY%ga(q- zSmGr&PDBjvlrCl8;ul<(a|#io=pYMV@{r??#IE-Yx1>1MGANjexSuoRUT|??i2g(t%jSVpR!XvR(#F$fS>A|%~D1u9l>wsVu*S@vmSC3`2NQZukArvdD%=H~$J@P19yuUP<%yUd_o zn^#DEjavY~|G)ryXQ&tLQ({o(wKfP8`zx#!4R`4l0l{j^5Zq%Dzr3}zlw76e%AD1{ zF5I#}a1GUHHAIAy?R9SAcS*5Hf1qbLS4z$PD2?!Z2sKQ5PKlYbOb@wkf#k0e>qmn( z!xEpAIIl7&ITVy3Q90Bov7p?VYJcOttn$~iZx16||Z$x@vO{9R%#MX`D&liht&3H^ekxZx6oTVB?kL7?KTZng{B z-TZm2jm72DyIWL@lOCF0JyuB)UQ2TG>AbX>R+Fb^CW*y`gV=hh?%X!frh_A1B^v5v zA0Q6JtK3`fPmZMIkm_nR6LQP>=~j-3*d-yXquim&=KA`2RjCxoUo)?JhN9PO zp}{|!1Z2K5asP~qFix5=%DT_L5*jGmK<5rn+%l%)rCM>(o8ysi`HCjryiiHBRUW^6 zQQ~3QM2iXocsVQ|3C&GARPHU)PR!-V^h^qU;?!&^z#dYLh4xl9|FWh9}b}QYD_77pt#y zl2k_Ub6SRARiFE?ajK}>@(*I#eI71g5f2=(~{6DBjdNCo8^7NO2dY?Zxl{AJhtDTrMgKJ&{etY3}$ zL{_k|+9mo}$SQVyTp^s6fzFA-sVcQpIVEN(&CQLUVuTH}(Fq$HsBKS(Mf=sLH?bSU z@-i=XIj`>vOY4vFbGMc+$46zud(yf=4hunpr^wfvibpQ4XAZdy)-g9-JH0mXQGd{C zGj925di=B6Hn@^mCkl8cT|0|c4irx9a$O-rXxT9Lb5jxP%)t5z_6193ZfW*_x5qLb zWgEm**Sfpm2>#m30MRJ2qkdw5^G4OcWUc79J%4=R%A9^#Oryn_44zr+`iMSiLvK(0 zl+&A9!`X(@Z#1jqY32$Dv98>q;?#vc6~$N;CPcUNI5bV!C7F52(#&~LPCe1CnM8L* z`F^ffs2IJeU!*!ROei>O4NfDhKFM@pu7tjWYS_?$8#U;4~Dp_5p@V#$49wld-G&vpCo0%kD&!+Qs<_+Xh zZGD*gqx_)Dj=|rAQo}tu5XKw(;lVN>w-%w-aS*-=^L~Wa$ z+UX;sgh*y_ju(ajG|-YyYSo zq4+gSAv+@Z0jY+t)LFyeLZ+Wht5IXK$Jh#b3ry9D7iL4B-JtqpP+X&4s+YUp$vU_X zxd{h!Y_Mj*o-jBDUY`^A1n`2qJqw1)8-=BFV%>~4(P)?iltW6$_)NDaoY8uEMxT__ z68?u@_GLYV5e+^WsLptwI-h(TH+BmP_$C*~W(L7?^QE4|+A7Kl{1%@5QH?Xo)z0uz zH&&J-4T~A#c25pI8SUD|+4KggZHl)%a?8M*5^T#BDuAUf?C*J3j(TYXXxIkfwl!+i zsjmLyYN$ihSzDatpSHldg!iiV2Tb~;t|tujWO}rdI1u#v1rFK2tZh@oWvx|KqG%=# zO>+>2dPmr5wWTpzyJwu0YTh1Csi2H+)uJ3V(&B~w(IpGs;m7uM#_><@{$tB0>>9^B zOK|vlSf!)z<0@(B8P(;=iu*e{^bg$6#Sy$imnmP0)<4JS?vSVUEp;-An=EUs7IhA@ zN3ZTZOq=0+F`VUBGU|Otjj*q>#Pp$8gz|DEH3Yz)X4GAe;XteYgg^qLJFvFhl9CKa zx0AU3q4vGfeYHq)7p`ycAbt17LC7(b1KIw?JML|O}tTh;WB&gE_V0*DQoeLk>lFf_GbJyjF@sSyc#2TyZCzajsJ6#sMUA97ui zi}RPRWr6|$QU3?Z=H+T8{qTrCTqdgj7Qd&~D0sLAke_=R z%v%&HKCUBUcr@6x;K0=j}+duFkA~ zC^{6b0o{3w+V)cc6sDNUYPFPgK{zx?;UVw04^*J;kz+R*Wsi#DBjco7tikAAz=KdNVMDC#2I%yE%oC>> z&M(E6>1zLKwqefbTVhON%TT>l)2kHdOY3iiz!@E2^pW{Cg}Y+*Rf zzG><8eSS0ldOdBW0NNS6g5qW>G-40>VNM)wEKgs~ctY|Drcqq~qrdc3HuDUsX5Zy( z(oz$S1V@O*dErZ#-oW}*dN2`S7I;FnS^vihm@HQlP4RR)h-@yk9*;(6dq!ucA%S5Y zWxB0INt=yVXTz=Z++iIpuqXP=p28!BZdL9&GsTCw`9jBP_a1KdsRqhYN1V*&>QyMo zEtM+}DTadCBf3+(!IU#g2;OConS55rIdV9#P@8o5Rh&9Uk|jRP+)_l`tLqBg3I;+r zL=F?|H)Q(RfH^>G@{yk$<8BUYc7bV**+0dB#c*9ore_yRZqfjs(UG zj4pqbt`qgL^ZJudWjxKK?511Ib-KD1zN@ZaUMwaAc&2(PhByUV)o;yVx12@pntebl zTh+x{pB#LvYqYWK=nnw+K^Pr~*@Ark#<b@?f`s&t|#NV1z+H{Br|?sq0a$1k!*$~iw>f!KX1VDO*%k2^l|1s7&9b`_lH z0$j`yX>!sQB;;Dl<{>C|)fUO%Ke=`!DiB#T*L3Tq_6Bhy@-h=V`ybtM4^FCbN4(YW zj%xE!vPHg}h_|?Y=yqC>wZmS^i9r77z5gMMxO&o;A$ zdUyMl=iAFD0zEgIh<3~5EmsK7(qu7CkLHR~LA2ZfUdrJ_WjGht>X^YsbTZm*F6ws+Vz)Z3%!4XJ60u@=QIB-K|GUE=v}Lm%{7 zHa?ZiR{m=NbWO9m31KrDp=WXvEcX8SW~uycu&VVv<0Z8sx-pEOhtct7H1E3biAj64 zzye|J*>!K{X1|I#-@Uw0waw15NoHH(Bjo+|&7ja5-t40svQD^2tw6t2nd5cCM%6wC z&jDY_vuma@qH5<pKT4JVB)82X3d2t1;Zme0`Wk zEYYDxkXz&|!^@dJ@V`zWcH)Fx#D7;Q#lNc*-oH;FBDPk4bW*zWK>U*;>n8OP0WAP9YDM+fAp{yKFa*05bVX53=N8T*`iEr56&wE zCZ^8ghx!wRKbqL#fP$|UO)~#+WCkC4*krg5>e@B;{&Wkr4cpVMSPVB#Li_Q>;N7^$ ztccPpzp~xmxlba47_HyXtq3FVZI!v! zQXkw2NApdYYjjroy3dH-H9MD{CEE?(Uc^t=sW9VOUTH#Y&(K{ij10w8!24o!o@;b1 zk4z_Q&59=R$6Kpzt!eMqlC@@4i;?x2!>-Cfn{W)@ent zM7q8)&egHtJ7}>(nu!q$%@GXMXooVaU6vfPg9fN&LYm?ou}n@DUsn1_7Je$1>$26$~*Hjt?R6>`SCIC0d8?#6h?Ljr$p;KK#>| zZKDEh38*+y+2LZjqS?Ayt#YMMyBS|{`g_`E*S-fAcfg6GW z!0&?*2)Dm+n=x|x(zvxw$E{_3+TNshY5j`3Q^X>NyHmx&YFl2+)99|aG-^PuE& z-H5x@C6bn}s!8Ye<;x{g3|~z>MbQrh>q{z*XcLZ$ODiq@!bS_TH%JcD##7ekE*u0C zhToTBIo-(+CTj#aDhzor_X-yN&W&m7G**{t&wnx4A2Qh9Io(A5J4(FhUi3VbEo1do_H>Sr1yFQ+$SDVDf^5^7~ znbgDvyGa^T@<4}r;$*njue1J-Sdq(4w8W+bdx@OlL!r@hp!QW}(Sxh*m#x5fp zwG8pg4`SbgRV~-aj~w7}vV&HCVBv^gQryAFc3Nye`K9vN1?p=dMW z$SvEKviFSUlMv9XJRHZScN$b_Qyu(;liOd+O}|(#HEPb46(2jf-3Fq=3U|?_Jg8>x ziKVkx2T#H+?q<8{)EsM?&=w1g$K$F?V$rKU=w|Ap%ivFb?g?`SnTCw@5d|nc_smSb zb*0+ETKKY%Hp?mAU$=h+n@!krIp)M+iJ|_=_&lK}1(V*k1sYpmy#nb5y4`k{t>4+SXrYiU;?>3nD z-k#!uX{zPRt|)Kdijr*?Sh}jJHj^Q+8icN?D5KY`mh$v)sNL2A0+gDEd<<*S`YZuc_HW_e-)VWZVqmCFkjJ#gQ`c z*V|9-gMPI94Mkn_khLXbYi%}$rVbKZShm?1u}zVBQO%z@O1AGB62x2DJBv(Cm&OpW z?$#*s*L{>nfTH6PwdVfkqN;qit9i`=cGPH(Az?dG7$wB*QK;p{<1Vizk_2obT)Nr| z$tDldv3ZexcI-eTjXXPW)oL4&H1a~O-KtrqhUHYMs7;Hja!`1e zG9*lqJtt6-uM4y;081Cvjk}cdaWthC@XT_Tlg6b~n!81YXDBw5f0td4ZKB)=+J}PH z_YwB5S)j{?mNC&LMvAV6B2zvgBPg$F>*VQEwIxo|a<{QVFl0 z76o@Z`+0$IbJdF)Utm*gSG0HcOnCOIjvQM@3EviR0h`W2YA8r9|GvbMR&|H4@=31A z5X#Hc08+~`#$H3r$e`#gK`r6;##pqKJFTt0xKSm;d9;cEx!a|d_eb{%Xce`+D`PcIKk?@TcnmyjMhT2 zf#G9T89|We&;4y=Y%)h_E&K>EYkiS-3Drv_kwUzasiOe}D^F2I;*mTwz4c#IhgPPS!dU38OkQPTwj+35) zzI6*N4n8qI(+rCIDU^g+orUvRuzu=3#{<~<#rw<2*9+|t3ku8+qzg3}nr#<_c=aXm z&8->PlSi-kCuZx|Ut5FXdu(tB84O7w0S4}GQn}1z3K_|zQ$_cAKiEJy;JvCLYX{|6 z6?33r7&(V;7@AW{1(0vUmJ*@fcn(WE2?;UaV}jVjz9|>d3;p&CbXDwE$rIU=RKj{D8me6O!O}oB$+^>L@4Tb59Ecv zXBD!xh-%PKT$g4F)C9%JCB{nHw9#clv2Bt`Bk;!?l1$e_pcZF~CQMT9y)5F_&Y7vW zx3+>E68XEfgFIIaMJRjXbnubF%O)lI(gxFncX(y&!Z`rTtUST!2`3h@lt{U`{;19M zdSQBKGUVN2HOW*JmVWqXo*g?e1FbM*k6`6kCOWheJR^iI=sPvQk7}VJrx~yjM(P!% zeppS^G>Rmzz)A~dDZ799To0wZTckfUz_GVWF+A83^`aj~XHUV5VUtmzsF%y%NRcM5 z)XPQ9T~e#FK!#>peEwxD1~21gO)2x{ER(lEqN1a0zIjx%@;-^JhY~qFFim8wNUyXs zC_Ij$AZ@>j6Few&*_PZ)*0JJXFY`3_$fQ+t$b;GMrls)lZIg|vjqi4 zW!|bb3Y6u`05^vLRPbqXrF`V=1oLh1@biJ{Va=S$G{sXtO2J4_qteDAMvz%UJ%XDIhl<^S0xF~EH#>#BGrwgVnj+GOc_mmP=b|r4fzivgu zrDOMB_O3i3gOSM0n-a{M)pG=du!(TA-tP=4cFy`cHCxK(L8E#VViL9d#H5tBgDSjI z*w~s}W-k=S&75?6sv!MiJQhiDlqD98C(I|UbZ{#7`S80i{6#3}5qH9LShvpmKkk6J z{k(34F7Nl)w{n*hZb&0}OY&>-8_dV2&Yom=+e8k`p48CI`lyp0_8iQ z^?3bpydq_Pj%@*b3?uax;c(NVv{YHXqqKA;?LKiR#31Trwv;tDxJ5fXD7+>{;ppY1 zs-xCv>gSunTed16<$Y^J6v^bI}U3G$%0g?R|&)~XoLH`$=66ogJFRrG3lhyyn#Orijt^uc*W*>{mNdf zG-MZd%3fo%YXoi&4a^a;5*7O&A~tH8E{Fkat+eeu)sh-`_2Aj7w+vFVH@?1tDA@!F zqfS_6|9XCRowh zbtpOagAk~UeoV2lg%* z9}{eH&MLg37P>Y=PM}Sg09Jxb9Zh88A}4#Ape4$X%*YVyD(vo7D=9>^-aPVjJvr0j zzGFtj82~76u_Gq>xqW14bFhD;O4WXcc_ku}DcqkNHp44ezj+doD9ja|B`@SrowmMBC{KhZDo%;`xhn!wvcp3&UOW`+F+!4CgBBl`KacD|+KbPgu#CWJQ@REHe(W zci`Le)<)aUme237-{FqO3#ImmK{m2c6yKAt4YK!TPycf$_&a5EGp0K+aA`OT(!b9h z;tN!-be>(EA&HgiMwF6%PN`DjH^m+I43P51DgT#Fsr0 z{9O7~TcNysY-J1uzCkljPx5NjTr!s6eZIl6tZ7XN9x>6*7tKVr`4#*#%a>cBCOgMk zT?LUUdKL^_<5@&13q#YIc|Pd!h@)1Llvj}jQD9-&o0J))=-=NHLP}>o*ey3$JC`Y{cVKq`(_Y1Qgn zX$OG@cng#{^$RT0krId@Y;)ib^Pq(Iq~+qBe{kKn_|Aa}$?{N{CB>iKaV{*CSNIB6 zaUvtkDWXGDTNi4S>`92(-#)|$)kKADDwlCwP@p+VRxwY!s#gio!;uA?2gW6WBz*o*t}b<0kn8*eW1o61b=XtN2k4}zKJ;iM85Ioc9+ zM&x#XJtX`UZV@t3XPyztDqqIl6?^SLE^cdf47`URg+=ni9m`yNq1@T ztCx#AEEl=nkRrt;#i?*!oA(yFlF;F2lVUx=jHsi(CUi`qIk544zB{_w5E z(n;bP3qPUJ=OsL}b@|=iP4U|ne!P_nYu#j?y_rIR{u+gPuj4~g&zA~0mE2kCrjGvGmfm`AwbyHNvg-eDuJ^uC~NY-() zyAx7+_Vu~0O&ng1T5i>{F>%IIXrQsvmk`aVMEt>_V(BW4AFq2)U&}maitb4Gs5IQ=5#}F0B zG#J=>zqfA>(wazDgNqS!ZG%BMYkk9ONnFR9uerUNFMDHP1z*}OtQs%BWpTwX zcO$bJue8P9<~w_XUgv|_aZ<*p0?`N`=abyJbaMkyiDz)-3$&TV?#iP)$IRwCYa`h9 zsc%SPc(tF^%$w9M?|r`N88#`w z<;f_f)<(ldb|clwj(3l&o$uTHVA~_W=GbVuyRkBsR0!u`do;pTUX6U3qjai z=qpe3s=>y6T3$%{A(wraEuhtXFGtv1M2`dTq5>k@vf^C`%z zFj6LN&soO*j)aNwBgfHHa2LkJW1wxs&M~~T2v!=t!MwaR+OOkRHn~x}8sau|7#nu1 zouj89=74v|i{!T$&mJY@k>Kzgtoj>6r2HJ(=WkxU{?@^efK4|3X*T|C$D074U_3z~ z@^~xL>i{V@_WsQV_KbcKMzVOb@FK8LN$3XmKGk& za7%N<1k7;D6+zKu-v~=G% zwjNb&Vp}t{-FI~Iuc0O995AM~!78CnJ#ToIs}z*jG~?f>#z1aZ!L`~z(V?L&n}DK{ z0#LdrsQ)HSBqq`IAy3N7z|VsViSPms!qv=+g|^mh8?Vf_ErH^Wdp&T-O|+8G?ajfIqEsLbb?Y%|mvFT4rqv zbcLsyo!|N5MJ|Vq4QZ!Brp}4g*>1cn>pu0sp&efbDw!qQ4VQm^q$4 zEzp&x3iH$j@7nGx=$2f!=A5R}aQTVRFF5Kgd~XF+s4~Wor%A0%rVA<0n@6`!n<$ z)hUv}c}3&lvQEe9&g8EUmQ3ba{Az}$e^vX5^qF|N_jxf-e3)JNpc#Rl6^nNyn9rR! zj7hn<_pU?Zq(oy-Vr9LLd*;iHRE_K1Ho~;d`ifM15bJQfB*lu-JJ8HxQP$yc@a5m%1QLsWNPTi)<1`bjOe^Wj|o4w~uvRixkrU&}P+ZGo{RhHo~>;YXG;e(|SSSwcGP447fx^G5xe zZN*ywV{4M8i@%n2kP_#~AsMTSie;;_f0Db?fFhc{O+ne72q7ARK(&Cmk$w$(jGZ#0 z>l?|FaPKNax6!LCL2lwE2(tLxLs-MaIlxZ3i+*hfoM%d?rQZR5Lfyl?|Aw^sC7<4t zGky?+U*tr-vj_yA$vHRR4R8aEX{$K!Zb>l(bA-(woVc{>F+d9IPTKDrLF=JmSVxL! zL@@?^gd~(pJcl-g*(DR1MS>HD!zH~%QPk#~sj$H6F`yE|FFluwaC8}Bq58BQ5T|jJ z81El1j9?zfo3{qGPPTKZ++*xIu!el|pm!T|?zyv*pmyL}Uph?#cm=%TX646#1MroN zO&!A0Kty=*DH*3)7{5fdxk=5WloB?flY9ZQ^V&?&W#tP!SlKuLr7ExXdCYT^BDUQ+ z-nfKDczx)j#q0-6#rQ(`DP6PUX8dT1MR(9;?I` zN&H=eSIBCf-u>1Uy@;xEw-;x_5T!bl9S7Ei7^EGtYUSO9$Q}fqn9f2#QTe)l*Z5|LaCZL2q%F1Ggf%*%8?slyv1{aX$L7_6t@#^mOD30fIH3+|0-MZ zPbudp_eGU6Qb6N6mV3*9y+w{oy{(N-b6Qi{zRQ)!mCBV%kbil1F>Gbgr6$MY{yWrK zH7uV>v%*T&6WVQikGz1MK-6B!Pto2DYKNk)&PVSjx}_fNUajW11%lPQpwY)}`V-=w zcgS~)8s17=At{UU3&&NZy3LeR>SVh=@EXmP0Ay7+_k96H7}!^jxjQ`W)-D-|hZjPO zZ(xWX#KBw8+d~OggjcZ0I|D~3KM0AvQ%3yWkh52UvOxz!d_UNjHc0uQn|!PnVc0LA z%6c66Qy>c`h*Rf~$@YMlE#NH99PBoEn^|y=d)Bo9zS|!PVG^DYpSczTus%ON6TJ7? zazStIbYBAy0SfKFyLb6@X^19cVU2Lt7QwGjsiUy#iuyw^TshjWnC_em<`NH1wh#

&r&pfX1~IF5m^*zu zx9mOWH+!WAe4nmeae0B8uNkk@0+720IJbCyRDS&oqXe%hyfF?(SfAFp!?pXtZ*E-y z4hNy15c)F?2T5<$KJ2?w7jJexRK3Aro)iY-HwR8%G=9{*VGesQ_nU7Hzk5GklYD6W z0lTvX2b*sfsvkK1vAvT9M+6T7ceSsRAg?q90lnnn_ZnhI-;I>}wNWH*IL0vFm7220 z%3mDy*}ZhcpLq3|y$myZ1d*_B76-;(32n*##O(J9!%v?E2jpK4cdXyyT?4&n4!1W$ zn?7-aOUJ?O{sKf<`$ zITx(LVJz~;JYvyMvfT!*bVZM~j0+xknZ~`!(=WS}W!>~iO1L;8&SpAAQy%P@7TZOp zocYPjd3KRUxpq-pvI)X}o~9Z1JV4e9PGZ0|-x(*M|A=Zf8H&(O;?%N1l4(*SE*l-h2XlRHHsLgi7~DU)9! zyJj`2@lotR_m8Pl&kim9l14b}V_lnOT)50*CG?FrH35dknL@36x1esRKM{NT{ZMlalB#-CmLA)|&sAKfV4IY-`DJ>0XR@K{8aLy{E|k5-d|-7E za-P(g6IVd#hOCy!%b=$En0a8BFCCNb(l<1aAHmhXoM zh=$PX#mAgvl|7@%-Ju=bFpN0u6_)iwK+gGq0bK4P@i}AiLHV(@`N=u5-GkW>F)bl< zp+2MTn$tmGI0v3TkjblE8rU-TqcxjBMzv3r4-wq$gAUi2S45lfSPRVy#$5Qp2Q|YD z0jY>0Yt9F}!i#%QY%aYkU)`d@0j)BV`(Z$U^ zX)F<^*bd8)PcO7rC!Hshl`}y;Ud#R3S~zd&-7c(AWjH^#o3*z=iA>uEXrlY9^ub!D`ot>rOKxFd$kH9&hS6Uf4&5JGFftAc;(`82p7>xj6xJjAnv zRg)3i-@?a!*SQtNzlp-)RnZAIHHq1G<_46)K*NrwA{|xy6@F58yt*S(XYsQ*FjChJ0{qAFmSkF>H^J^ywghq&V%18E&_hK|e%$McSPT%>WWqhMxX{F`!aXQ*yjvKC^=rb^&}5c$l7&otX%F)V#sGYKni~V@zkzSZKcQsQ?@S#je?zNW zPD+}r%JeZj1F1x@VF13sN`lrk)Hp&n2yVdKuW0zC{ITD9>(5WsM-(!D!P&f#PySrM z(3xAk@()Vq`-#){;a`!i?n(9K&sgO9Q|$ZU@r!`M^9*z1&|b28_Ouex@a@x2VCjk1 zAnWEqhEj2-Z0FJ;=;pW-G?~$s{kzwgF|>O^oJVgi)Wwm$?F;c=Q?VjEuLRnlfPi3r z{Kr(`f2LxoS~{B0HpTYXeVT{PdK&N2-KxleN;uf-seD04m)|N%JF;VOls^MYB56u=a57;T>+Azo+#_ z7=H6NVdTq3}*K4u?% z0K?er@BvxO%kJn#{BoLUo5ry3YlEZ#^g*T2R;@8)Pszc>8r2}^{gDtBzH&V*B8{i0 z7@d2XAtFASRV}Z#EWno#Ie6$I90n=urnKIR&bAilN^fBwE4xNS_ztEK|gcFm2O=@T9Rb9XUw16Nz6af8Fr%J zUMg;#nyL+n)G^Xi&uEDsL&OOkx68m{qU(#_DuOWeMWAS#4`s!~Dy^_4S@`tCD@1Oj z+m6#tt>H;OymcRqpz_ zLqqY^DREMB%&ubbFYFhS=m|OE4AgHUgEY_{os>@o7cNVJ1W-?+?@F0dX&M`cI+JRS zUQX+0-XG`7=v6CBdk@?dvaHt2%}MedEF8E>mF2_MO=wr?vJ&FxBx~{;b>DSY zb&FwQnYWsfXRV_KhqeHz&?AP~-3sY7psC#{dgj8mqf&6Xmv3uEqju2YJmpvJasOUCr@O0Sr{naqBR-Ca_0JGPo zUxy$LvZpj%mchA_Ec>!YEqHWW_}(9; zy+d5^R9as!r)@kjKTzNu4p*^vkcxMq8u;ORQ{K7&yMg05uL6lVE=Hb+`wLhb+9~x_ zKK~D8@4%f|6m1KpV%xTDR-9C9^NXD?wrv{~+qP|0Qn772)#-6>zaIC#+i#2>=O^s7 z=U!*dwdOpHZVyBmXjA&wcf>BQfnlpDsA9TLR9e%y2NBU&k8Q&PM(Xvj@u7cFUsfeE z-2k0zr{FM_L^_jLt7GnKH((6fhpP}SIaXZ<=Gs$J7(7XhtNseLxdunU)!xS52xx2Q z>yk-sy7t*u`9J7BagIX~wAMQpJ)}Em$*#AS5)_6h{KYY^E0fDA+{@j?KiNG$7a{u@ z@2NgKCe$#67_(5PGeFiIWfiK_lNsm09C?2k_B(6<;96q&y>ZqeX$&W_ zRYD7PQ(M-rPRH?`6^)P#JGYAi+Wo8Oenz&x4Q-n&OMN^65}q7-Fgoc7BbS(T(=+t~ z##yHsrkqxbp?%C%Z7B9c7wLDD;Zp9b8cbYT=NykPl>yeyLRIh8fCLr}|wnH{qYqB|QE$@amtk|o~qO_fGEi`5R*FnCm-jAaqkMimKg zh^7ik;%r~)2OcZ3FR#S>358$;!W})K|1`nvp8}Q^$2G;~v$SnV=v5jwJa=>_2Vp}Mp1G`I9+*zz7q+)3nBFx819T_;QA(3^?$|JT z;fF}?Gk1)I`RgxrOkwriWj3~B0Np{Qr9yrwi{yJGS^_q@J#Xh|H%6J zB{u_D{(=5%WMjfm33;`*BUzH&*W8_^e?8m~=c;1ykX139^J^;y(ofLrKB6HJ|LDNO z@}QhD%z~)|V`2aDiQzezr}RCKc@Xx%Albpn<{P*FsKE>X?LWHK}89K^>| zB+lyf`ofCqe$9;T`GZyK)vgV8z*{4o@jF`TU%8ATAH%X|G< zdZKQQizy)@BKb6n^@J5*boOVlYw2xftjilcXA&qrI-R6GdxFY%BhSFnzLW(2oQ#fo zGIa|sL0J`JwjY4%)*b%zReJGGzo2vctaPbt%k7Sz!RQ0um`F7w|72X*O^MZoP1 zn^chY8W94>W|F?Pi%JhHWtALS=iA^?{bdI4A(r+oE_<|0-=Q0HHVBC_q&;6^tql}< z8;kDuWgFag$?-yxGiMR_i?_@|8=y~?LIS-9M@bmGcP(zXmmoY?aq?o7|2Bfrrn8@+ z_&U3*p2CHK?pA40|BxNMZqxFC^}j4Wp6446g`6bwbxx=1*e zA6!Ee+G&x?KfoPotE;Qu*Pgn=W~*D8eJ)ihg|>LMyE9}Hg-_lw_CHgddhT=X1t*_< z?$2&Zk-+-2UK;RP21{{naJ*{_$${MN%m?}TXSWP-bb}c+_IysBAuzVAu&8hn0fV}8mrd1K)#U|) zK}b$gZxEgfUW6--je~uI8NJaxJ|}`FeeyRy!o5dHS}X*8Q@mjJs}rum^Za}vzel(|VdWN{0P4izj5{7Dik*KkCJ0!E7q+8#g<;JtJGg6?9!R#K7(w=`QUEB& zE7^x8m}JK<-nAHjoZMuFn{^zN<<}W?f=0w2406&k*u$?JWaPZ%C+OylH9j0^Y({f- z7axTspgQB08Xf0Vx8fc~!0Io};KrzA&prx;d`afY{T|omQs0-&?AMa586R5Z4ij|Z z$o>r@tea=d&<|$uvx!WB(*~c`n8!;nTZ*HbzYoEyKemZWz_ds4li=LN zW^V2}RLy)Eaq1rLsg$U$yaOpgaZo zm7~(z+Q(Gbpsc)#6B8ylbA-f14koOC#ellyFIKb;^x2CLHFBj85Ccgknpgqm-IBAE z61m}MDC@j>OvJ!v&IqszX#lKeLfxrnqZw6)EPyUuVGqul)b^np+-+D6weVR@20Aq| z9hj3e6{EzxWmc`KC*x^1Ex@)YE?&R|BYR@4JgO#f2pcgpNd*A49ycl;5!7Yqkr5Nb z+9^;eQA$@2-06|&D`AJTs>5dia~b(8eY|Fzt&U+-zy55|$Q;clV?;nyQCG*)2gZHC zeLI=RP9`^V{><#{AvuX!Sm;6*D7{%~D+)AXRO=@Um#vniF!sM`5n)*?p+%2sLY4TF zACG}zRBfSLqv+I9YZ|{}!~hnA467XehlP_DpCoe)yDspHny)rV-+DM5y}sa<*mi|3 zgnbZ8PpTo}B2tq@qA_cxnZZrh79+0St{aS+-Q)JEu# zYxOTya^4&hG;Cf!f%WY{CfSP86raR;bsT$}Cel>DfJ6GrO^zoW9o{`ZZCK6YL69nYfR(?%O< z+u~E^j44xj7)a_;_O8Eu2<|WgW#YQPF3MU?sb}SUFPd}xOw!FNxTvn^n!KGXYqxG_ zQQ1b5-0|-Lupc;Tq3a<9ciB-)L}*tY?gslCW?O3nia>nm?@5JWXT3S1A`tQ zH9ecK$W!u8fpeO=atiLEJru}~&ey-EZC30QK4D)r{xIy0BT_s*k*F>LODDKC;@qS) zTIlGNoUCli_C6Kg%vk;_#BxOEqbJ=ZQUr5#Dkn7?fGQz8iK1vE-wc%lZ!8D3)j@tk zZ^8RYgU@n4WyZt5j}6s&VQqlmfHguFVylX$Yb_$AejL}0UC~aAdr|=3&FDw_ zYM;)H`tl0PanltY-F`q`*(OhvNTukm3(4Y)+J-Fk5$>SsVP{IXv;q4Zb8{j>Cqe#oL@hV& z7GNjG9qSJl*!9RJ{ZhYcw(p$>q0>Ia(JhZu>bVse!*Mb*&bTN3-eE-ox&Gv z@#_NW(n^94ak1imQ5B>Kz#(f19Z{BTPc~$Zs16JYqr~=9;_N?3t%YOZx>72177qF= zgAp^5yu9!qG(We3B&gy7k{p=IJNhDr$w*)!1=YlPP2s%*MPH>pG6EC;frtcP&kp?8 zqnEgZCpU5{jT?QXt0~<5eL-zGV0XwkgD$aWvI@q$uzAd(P~bvnA-bNOj-5neIJVyFC1IRdYYbX zI&`1AmfaYds-0uGl8gX7LT)`VM<`G7v2ijq`!tK_@C^L04XMMY+d6g*xRPm#5&SXNeJHv{;=VW{{-QT8cn{LV2*$Qso%JS}X+BHPW1%59Pz5~N@x zG#Ztg-C3jkA+pfd8`ShyN-Qoj1chkU#AU>(4>9kdmpYtC<)swEw<)aSBRK)JCAlW& z_M(f+qnPxG0{hkdfDckmTqls;)`C}bp{M;GcP{~Y_+&t+k%JEz9`XXZq_k!7_}NWn zCzd+zrROFYfB8}}lwFV>p6?b3_F$N2@3BV5dn&RaeT4$!l8Es^r~&!uN<{v3Q?!sh zM-*pVa5J2;cqRs+%b{M%xbd9J20HZ(>Z0SI;4j%QoI z8lEM2zk7VSLY(j0SGw6mS@Am!_2Bc5D-3$BMGcH3(H+d4&Pqdnon1~Vwin?8dVR@_ z5Bohd;=@ia(WI#WVS=-6p$EIiVx{{Nrh(LHM^=Z~|#!Ea`ltQ}Y38>d^TI zhOl}d=Bf*)!=*Is{24V%MhI@~4Sngf>l%LPw96WQ>9pIL{*tSPF@3oMVfiiv$4CP1 zzEliMkpzSi3bfqWpQTw`ZDDmL9h$T4gOjq z*H0Vt?P92Ue?S{{37)3@M!_v*ADYkKo*X%ph=6BueO2)~1)GL_v9kwCtcL>@Cm z^lJ$moqmrwa$U%SjpCo1whn`q^- zjRf9v+M+R>L$z|9vN0?Q!zK+psYIzbM*zl@QN1}#y?IfiG^TEl)N2rsbJbP;D;+$W z4#>mx!+0D!EJK6P-VzMUOwfZ{epN(`w4vMB53v9Fjm8SVFP?h)E=^N_Sx)4xit*cI$(;G?amrP(@N*PlUw9uenj^mRSr7O%lA#;kHI*fWU-v-*>F!x!PUaPqexD z0h`PXux|ir$x9I) z>V*Cvfm|y|ULfiY(5gUtjxeFQtd;hSBsY*ual<-2l2mavXAcFTC}Y6;F6XH@U=_Co zel(%yjBvdHn(f!~dTM!jYB^Osy|hg|`s9Z6t8e%^WX+|~P6r`m$H)Qe`ddy5YIPO3 ziQ}`;=m)|Ue~YMQPm0&T9i+W)IeZWX)b=dH<68e z-yvtFy<4axXw8k8+bAO+MknWs#7TW|7@-F28Pq>Yfg0X&LO}lLACWj|`RGntffm@a z4@3aW-=k@16c>c3i8ER&Mc7YCS*mfp`=+FTGm&dqx*sb?1}ikX@6%qVto2yAAy|fG zs>BE>p@Zm_+3Zr8po_)nO~M`Up{P(fWeZ^kM*ii_stRCSvSHroKe_>w^VWHoM}tP1 zAd>(T%{p-P0cGfU3xp+MeDywwHjtX&s1sBbg)m>RkTmg9JF*MPnRgNpqyei1QZdh= zUrCtmFr+s+;R8B#m|u9yg9f`s+=zI{yUNMCtlx7y=}1}iNV)-^hH!}fl|XFWnBqDF z$w3ZxMBT6?L%!v{$2MrYp<*D~azCO?a25+lt0A`CAoRshGmy{DpAX1(F~YY)*oF7B z(+w2-32yrW%^$7MSNMX?9|~p2^o5!`s3?hWtV};7bA}CT)7O-qb!aro`)#h`i!Hn z9{BQ!rGR2xnM>Ae%9VZRSEKB26M{c?jM+5T{kX46*;Ilda0grt`5zdJ%6GsB)*orK zNx%$Nur#{HI{Azsy5vLm{5{d%sA4h^#z_q@U=z)4((PeywtuV0nTxtY0Ec?Ih4}qt zcMu~feukBf@LXlQ`W2Wvrf1A&X}f04piCO)H?WVOUCMeP8kN?-EKeByQoF&+Gcfn= zH{>ftgJGSe*;~#>nT>HblaEZsQYK-<4H&lo36wMhHY z#aihH!jdh@`A)bEM!|<5$zP5w=RO_sg0OL-x#tjN1>L{>byvJ6MyPhp;}6%QS0B-1 z8ua*Rf}Z+)#}epxmuWp_)<9pq%x#%o`dNg50!xo6Xsw5II%4qmu~kuj>z-Zt!NYRAAg#4t zJLJAAAbiOLk#$wxiYB^q@KjFTX(RkBIV^+8ip39tCAl7RV{XoA*h4=wT_8==koMb@ zAe**7O`w`j=#Se`*ePA=pivCuMj}*U`|o<8Gd(Z8%C|4b2qXvy*8fhNiHZCtU#6;K zk1CA*1(~WRT=zh_>{+eB*o?Oyg+h)}Sr6HaBD$x-MvwM4^eS<)f2{D|Fp(jeTp{Hb zh+p#Ug%GtPP09m+@|e$S#;u2^(aHDU>n&U#n9WEYnJqDzDD){c=@{-9&R;|5%N88v zLxHb=ECS2cDS}yBIEA%=S+*01^?K6;lI$5(oTgGcJ!bl3Tgk4giKCD+Es~or{F2QB?m`ZMfiQr=L}@*mmg1W>XMw$RMy-2>tL!GO z^~Ed|40S~-&;09q)$~8{`hNy2n@-v&Pjg&zWIC2|{&g>{{+lZ0p;5S}nzJ1(JrVXISi?{mXv12plo2;j#HJM68zocH0a zjR8TUwbhGliJnkFvWGYll zcbr5*k`NGoU!fSEeNL245KZ2LFKzNUgw7B%2cccrNV7#KnLkfGimR2Tx@;i{$a?nYesW8lmket3P|i!)$Kxg18`5$Tgv!T9fbD=pl-Ch zA@_6gm4f0aOWo|QK4&tX_RlW~`hIY#{4!D-&rEcoz@vnvjD*h+dSSF*2_b4*=;RRu z7-GSzsGUNPJP+{NyNU0$ELi*C)~babB%HScLP#_!Ht!|vTH9BeYDKmmGsE~?APZ@) z!3SlhABRj^7m>d>?3>Dys-?oBxmV|TtZ)I72-xb@u@`>@wh|Yvc!URk^8EC19TNSO zfrv8|3k|xp6z^98f1AkKA_4xk*lzcFCzWxQ3=`%Cws$BY&rF{>^6kiAvf?(BQF&+n z1-_|*XIaEjOkPS}YZ3wQ_;|!V^3Cz>pe7aWg~`)PPtSWM&oTPdpO?x1tF&AtUq5ECIrqBh15yTAk5|;Djx{?kyaluw~3$r*I)CAG#Ri z6&e`JE#PmN;`SbnL`Lt(U@ZnNvS|E#C zN97e0TL_yA;hTG}mSk`$Cd%WA`6kr9D4RKt+^%l#*()k2mw+NHWY|4rf6nl1w_kjH zd_MB|L;X$67dlK&b0C6awIuB56pxj}k23|5n+D$olTnCDWEw}1beW`wHclZ&k33-) zsa58&IUm8RZ2#n)I)ADeOOskmYGha1LTlNbIF=r#!BGp{%{C*bpeiuYI*U-Bu3Vk! zn8Tm^Da95n`^lXJctM`NjEWOsfNLzf;%)(8ej`zWaq=Io7PWcX(YeagY*Q>XB@}Hb z4VhR5cD+CJIvQ@9)j6>ol7-A&@hcMr(l#ATLs~9s&J*D_{~$issN`z7LroX7+Tyu} z^V7Z!6Cm3ch)0-r6wn*VoeyDcN5SJ&0*c~+y_&5-iV~Ji6 z)|whOpm?gsl>GCLH1&aU&eZM^8on%Bs|6x1&O8U$9bc=xGuP*tPEu<2u%f>Wu&_tI zbUblEww2jr^+_I=`I)PY*|IE)gs}D^88GV0_|i7S3hJ%1R<%nu+jo;^#Vs0~p+XpE zO0-;p%avx|7(HsS!h~`#8r-@ZZWfAG%g$zoE5r)EM9;ULhRbH%H=RG9m?J>C*{_&D zju4TmbI8jdOdk-2XZw2GNA;RQ1rC`zEY;3H9`e$9s!v1(^L6(P;-O*4y->MY^I<*gjYvU`NaYr7 zq7TO{cmyIl$Qq`yJEf2CSHlsuB8a%6If4WY0RxLj#4IGU!#~5*fT)Go_m003e>?2W zoJQp2K?^s??r>fy61tswqX!6r!#Kz~Z0x5Wg*y)QaqlstlLU+Tnw<(BtI^*6Hx+9n z#6dIVI}X8${Qu*gXgC6ZK-2%UkG$0K@>Y3v`S)cb^(@`vJ3b;z0uCm{JTf8&N|6vS z^w*JvW#n%tN+MKIpXOSG(YHAGYbqeMJu|;Btou1{Q?X>EW#`$m|}E$Ge^fKT)5DH>Lmf;5FWh4G8Al-3u7n@1&=|6Ys4dtRkQO zA*|w0Q8YjDOQNio#3rW97c|R%OGCe<-j5qT49510L3*NV$JRB#@ZHCUR%E>e{OxzL zeso3mvfnbG`C;@K?AwFB=zQeZJYIjkKz$TO_$5C6TOI2CjyXs$fZo}@kp|fwafA2f z@>hGw#Xo%iQSnbG^(8+-H|Z-A+CK)~e`GmC87~h0gMta8*4aw^suQYe z`X_>abZ@lO7?6-2Q&v=eS|X@qEWF8KFssJo(}$ehM(DxH1=TN^tOT5FsaBi|(RWr> z+KaY|T)q?$Ov{^INbzM%(l(`8ry#o7GM>);#hx`SVBssQvtUkHUBr-+H1@iz>DFGu zkTJceVW}3{R2-2nWo_@bn2{erd?;l_Ig?<`K(GEIA957TMhf}7hUQ65*SUqYNpzVx zF^r~%SGd|;=o*{!D!K2KIFH|2P-oLJqAi&mPWkK{bHsXkkbCvm)Bzx8?J_m7@vbwc zXI<(v?(&-)b1y1&+jloH39cO^(+MT}UgAIxY`i=z-%@4GilTH{FN~n6>AagT!@G`p zc(%dAM9$^6GhEF z6ORaw5S?O;2p{9PI^GF;hluz}z6f ztH;Khm7%0ri;pW?N{I@ql}ECyIv5uV>Y+qcJHPmjnQD#Aso>Ri@>uyX+zH#rmeuml zCl}2W9m(w9E!nf5JZTnYi}OVT3o@QI>?7@_P=!39{YMtsUc-vbvZV0lAwmaV={YmQ z^j*N1n333#%?X%=g+R5L-QB6$&!`;qIOi&@6 zQ*se_6>NK@l}Gr`NBGW_*Ud=C^_vcSED0F$Q@Tl_*%q`3Sp+W)Jwlv8tWXOOYxFrh zi}*Ns_D@9^?2`)hyFxRBW2+e!j48Y9p%q-JLHW1{7^*KX=Qp;)o82fnS$M<>oRk-x ztd#Imav_(lvZe2%YeIi9q=zZNY*l6^Gp)jaIZ`)0 z;M(;^QA5kB2NvhdN`^skh50k}+Hz2pp9Nq}->j%NIi`EIg^{XVR8x;U$W7C-#8Vss zbLyEk;bYeCI)bmE`S&t<+|~#Xx0n!Y1gI@Mi60cf-^X-JZvYA+Vpsf8j{1WNciq=- z1aY$L(dltKPH>{?zqL6pmioM*ETdOi!?lEvs$H_is2;*oO53zqq3FU4qhr->#kB-0 z<5DL7rc}+YnpV73WT$#U?a8xoM!$YGvosN+V|u7Apk>X!PW!bx++D+MgYL(3skt#Z!rf$Ez9O0!7slGU-PEFh+6tJwV@ zPE~q(agBo={V=aPPd)&ZJ47`GD2;3cYCmU3&JC!%pnHW#JAUaFW76?z{pxKL2_Tsns4()zD4`plgtgUpLm8AE)wVAwDzsInHaH_IJ7Cvw zz1)Yyw#;tiLYba;mdSTsg4O<&^~Rc@;uw3zyDZf@&2|w1wSW!c6!(5A_8&l-S$q&}hx*Q>Mf{)KUnl{2@|wXzAIBRWQ%82Ts+^eZoAZPjX7@#b-MVzvy>ps5PxnI-;Cb1%AU^H%B=h_|M1NWI6-bb zVVLh&3qtL~Pd&x5-PDee#Y;{jyU?F4xgIHDRUMPXg|b;>lOSRa$*~PhL)>=Z?4<_2 zRN(BrLO582zMKWR_ru?EK->;2u1O_%*HEF~J>(JiRtd4$i9)v)VhFSm;>|_qKippH zg%bz);t%WGfOGEAL*7u>h$Jpw#zw`9X#d+^je5ZmseI50%~N+sH4sg`sxC=BjM~?& zQ5^2|@BOrVIq9@!vZKp`Y8fDxF^RyGtZxe@dUqfyRZ@m-U)(HuC&tWXj1&JZ-q$AY zu-tJ+=Z%cyCHW=PDusR-xgChL$<(96dEPQkzD5Q+`&nZtpyf>me~n-mUNDS8;Tg8w z+?{FUUj7Um?(Z{#oc+cyA6I79-(K zuG{=6-wuu}osnGTtRC49J9>@EoScqd>%z3*$~k5eAP2W%<2BB4NMSr?ejf$I?qS`K zREZG8B6F1rjEnlB{uasoq++gq|FfpVH7{UD{D;N;h{Z6Kg<72z->x`lS;?60mS5x} zH*ZF&#ef)oZ%Iw}RDH^P40eRb?D_}(-Z~Gj>Q-0wGfx*_NVb?0Mod&U}p?>c1=+BKCrvP3|pA{^c}Th&tHz)P`n}msN3(&*DDFTK5NI7 z83)*Z+pS3td2wea#h7nqKLomiOZ7(zYmSsgxm$uR!T4)lyvckP z0>#~#2oK=l(m~?xre!{PyW;F2$y)qH2Zn{eFa-35b`@a|KbZ~o%_^BZbyEsa+PF*p z>0+X$t|F4F(LqbalTqpY5(y^1Py_sio4^%~2i^%unvW%ot^s4htDbf`6y>7k_w4JT zW6Pq5;~w)ffqG+JKcYl`O5`yA7Ge)O7wu8(mKa!SP~0r9Lt!20iFf$9q1d0702xyZ ziEw<{!soHSAsUX$JQRG`6z1+<=pO!{FTM)ce6~1Ix9SlPsg7gz@I_zQQ*z!hZbOM< z$ssh(FGL+Zqg zDLecCE&4!;N*eBRMx=t=8|QDYy0ip~$KjPZr>RwZQ<|s0?_mEi>H+JU+71r8 zNFo*M^6L~BhFdO?8@$N;^v>=6X!DSF|Lqrnc+$_)F*<^Sx0ff7s;75R` zXSN=sG^Gt%qMe;NAl_!3)t=~@^m1~yijGUHEc#x zw~7jN9}YgOR`zJVB169$eW15s2@QGLmUDv0qByMFzJ@NW9NYfCijwgr$|T8NfhnC$ z9gz}Cxsva-(pZ=bM7Rvs{FdC;?f z-ER;6xdh2s7IR%*z{l#YHU z&Sw(RQGGqJwr)IpHsjG3g&2Z+Xt#Mr_itm5cYf>I**e!Zdi%&(l-gM=r?25I2K(M! zBbRVOzh8ww=-zUBaF2HeAF{D^`||_k>85+WrWnxt}q@r9jTTBex za^OHrO9-khA;uGC!8BEXX~!vVoZPNIwAEUWTQDfY&k=1ol7*{yF9#oLt$BN~seLpii&y({ zF5#%~GCh_(iH$n}bBABP=npe4Y`xZGTl}x2zuZb<*J3OS^5{vAFe6`NL#JfJ%4+0s z%B10@6scDda#D=lV(L%R>RBkF2pA*B(Z5wTrcQZq9RHYNvuc`*9XW>DdSRDW4zhHJo=@+#}EK2NR90KGKeWyiwSeUkb`?c36L#Yrs)(%Z;(&G z-eR`@cfZ@AwN@0RgUh?-G7znF^f$*uNFb_7s&#YLmIwgxPW+zq(YIkRwU~(`ov*J3 z=&2YHITX&iD^PizlQVyyL(57gN?KZ-RZp+a3h$jBZcp9*wa(7gr8)NgQ8qJSTeTE| zJ4xANH0Dq9NvEU}b^ zGy|&FbMSk)?)NLre`Pg4FB^P+d>i+_ezWU*|2w<>|0Wj-8(Z7iyV;nUn46l2+Pm0& z=R`>UKe2}Y!HVU4Tl%AEpnoA|O!YA*N5L>)>70m*F3B>$sv(Oraw7L-(lg!}QYC>! z%bMvXMX)gSBn!^>G}(7yVmSP%dAm}$DRpjce$S2&X}&zWJG}6Dy7;`DY@Ox*cYkc~ z1AgC{K##k0w=0%1x+xE>aPLJKA8W`#X%u@T#8qlgjE(Vp&OzK=A#%S$-k>CSwkZ?` zp%K2CZgu#yI7R36Vj2fBRiKGp_UIS~eWkS(XQ6$4S#Po`G}d~p$6O=G-z%Tyr}`9L z_Cw#@0KVQR_EP44$_K9K;+tG(+j(+MeARyJaWEQq#snfmQu>`tcD}%|yLY`d<6WL1 zhF*g+O7tP4P5{{zLLsUh4@T&G_GQ|1NAuyk*=7y;wA|HgAM9sec2lcB@6p`?C6nB% z8q-04vj$N)^(JPEv;~B!Oq8|Sfazp4sv|20I>)j0R!K&2n;avkcD1O`gn@$(tPRs0 zg5XZ0yduo~0&1!Lt3aX~S=;i6O+l^gYrG+56<-4a&x195L~n6)CXv@-mulDaW&G6i ze#<(x9z6wR>G7wPp$du-bI`v(ZW4Vs-wc%K**{k5k6GN4k$1>g62K1F1NMoy$e;#f zLpoc$SYZBh-7;SCGR=L47iTXTZ_g;OVdovpbg~jjwVN!69)Vjw1Zu$CFl&VlC&6Y* z2oVkvQ+!R~raXW`*PQ#Gyeb>~r}f5q&nTi7wGD(XgLcbI>r;Sm(g&k0v-9HD*Zv5( zisaNQGq7u#r9TZ5@-f?6Vu+|4y7^4JRvPJJ4%Q>qK3rVdv{6(GegW;WV1=->e(^!- z9Alqym4Z4U5t_t94Je`vfd;?mk!^?1;(6L<*l`U)|A{VV4HbSVDkUiNfn`c$)uBLh zDY{NpO6mQ}oQnc%!8un1N*?V62c9@6?tlwM;#pY8ibzHIO$E9Dm!>z~Esy6DdJT>J z-vh2z%9B!riP%pB^3YlRlqvOclNcL5AEY_Qf2jFopMw)|7GAJE)s>GRBW#u`u;j_3 z7Vn7Xq$xMK$dPx`;fboT|NPOU#IYs!1_W`4z6j44c?)H#>I&;N)0h^d#Gi-&-K|ki zQC6bYO203a4oej!V$RC*8BJxMSY?Jl(#lKe*o&obzt~+^Ww!3WC6EGR6sM+k2;EWG zyEbvoSknP*@o)|i?cr9D`oQb+*6`-v?cd+I1+-nHXmg~v1|}UW4|*0C-$Yi@{&_d8e+}oE5PvEnZ!x9z%n5;JT0)IN_l;NIy}t z;H797gK+I#q%0~lD{}AQ>{{Dw{-UHaYwG~Vk(pATuon!3Bp;o2!Gl@ix@)e(eKcH> zKPz`4jkJG}n)mm`2XuviVlqSnvH_RdSccSJd~1y1M=yiP-7XodbLcdS0_ zhsr`0G-_{nOXQ@}k8@C=6K>%GB)<3MWns66isNU-q!fR)O@<68nkG&o{aij{^l%N& zeV+azswqg?@Kxke3ktpKd%F5(zt9U{6tG_Z*T>`?lKwL2`?yiQRTQTG^O66L4~UA6 z0@}BP8;Y6AIH@M|gNdQ6NnBGL8N*oQCXtdF$pVeAbUmNm(md;wKESUyA}~R0`%`Iz ztrKnu7UtIUNVoTXn#bJt^Xm(&&zQfJ#UBUG&DMEWAM&%!zDNtt13g-`iFqXoiqn(D zKFO;eza}IMlaNJIA6c3KeQ;xhaQoPVgW7%Rk>3a3Srx6rJzzQpCex#$5a1h zVTB22;Z-?bcEwW>13n0^2mN|bLc9#AOPOkNOk(HOGlOpXB!Z>d?K73;6awKn{u(mC z>!g|1PVQHJS%KqqW&CFqal|0-~uPQnH9w)$&7!eiYX`V6BUk|7pSt;>cey zX(kt-z+H+NlfTFNGB<3BdpA?h;)CCB(!`2o7YCZ{*v>F`W>!^WPK?bvBF&BF4YTD? zjFV>I${lkoY>~+MfnqZf24N`7H4C^zFKHABCe_0gIwKN`o5zBd!Wcpt7sZT9IU7Kc zo$K(g)1&z9H!93^3MBG`el~&u>Jj>5Q9_r)M{>aX<@6Wy_(crNUn5!$u38Q* z{!uCDr(fujWE|+CbA0}{;H>(+x3Il$2wVM@d^!G4^(eHw3JTMD^qPqB`W|-9QG;x( z$(Ou-!p7+1-vEQ?%2zxZlI2ZAt_ad1`#sRaMXE!-LrhFH-i5mkuWzLA-}!`D??wX+ zb4Mnw$H^5*F=`c9<*M`G_)m+oTLJgP$WjXzuAT4-n{YC&7L*~go7*Cl+9!v8(4UsM zPoCN3LS2cT*pE(AqnUesBiTVXQ@p^rmL1-cKI5`njHa9T{$4{ssq(q~q}?v3(MCfOEoBU{~&C|XloiJDM|i9Ruo zfsBYVXWe?PZ?x7D6Hbc8?%pJrUfT-- zT@rgVb>Sr5`ndm=W5asXjltlo>4{ zR7k)b_=5S;5dn~mj$Ru5x?o4576T&T`ccXr3LU=WtunBzuvc&|SC*j+c{AxHb_$ry z%2G_<*~U8B_4LIj8^*RWGN}ZG>Df(@;sWQ`sXVmEQ`jCVP7^JR%qlAuHjLJ`Emv9BjX*z6I(dOB8H2 z*+Cqvn-QU}t+smqP8!2ydV+O+mSQ#J4zQ+$T^!>*Q7R^=Oi z1fk#tE45$*ZAJ}@!rKSSx(2HNs*VIEZqTo4&nC#TgsX#{F3`9Ni(v!2T)9UCd1tuY@02b-$KKQww!VFXc0`?Eh@=q`F# z{g#H1=0$cG_oqhR#`##qSilW$EVe8V+-9{`rN!2PeMV>5tXxo?R`bw^(M(&OzLu{( z(C+*b^%IMhu~1TaMeRUIk*8e%G-3sc3V7OUJ>~jyla#;NGiVf%*$Q)h&`DrbS6Y~k zp{{z9NBhtu{fe^4w9&JCDMz{(9CF|M>O{ZA`XG1_tmJ48!e)9(3vbkOdbsJpDx&PL zDAt0!tVzw)JdJx%5(J98?>nrgZtJ0CRDr7(AB{=!6vIHnApy%WS)&;Wuv&@izPQ;v z9A+!SBSe6geJ*}=H6R0Hp>QEW{eU?SHu&?RFJgV1E3=fiY579c%0 zEA!4=Q-JhgT;FqN?XLv2S?ep+p^c2xGOxhbFUv-6K;)ycd$5OQ9=nSQDNTQ>y zI9p0kn?6F1>_(3jaQkNAkK~b5T_4V z{(o=KSB%wl<-dUn^Bbtx{!c(9?hdrK`wri-1latq2kt+xDn(828>_H=5d&-z?E>!P ziYcm>Ly5ubn#85yD0W4`q@?l;E8ECrG&3(o>dFsiqQ0P+y6#HSe4`;pX57!C{|(~! z1!4U;wrR1XESiB%a+>kldiL3hJKo$B)c<_L^G8xbGz7W#eW~Bc*$%kJ>hqz8L>8y_ zr=W+b3si$3@;YwqaeeglCQWNot$wYK4GuH`$w`0S4cOp1NW3Fqc%BAnxl!zI=>yvE zWoX7#mug_`b&mfXDc~P5?i`N+FMfy>()dvHst~`+%!n$wNCX_2LCi2|A0y3>g-s4otgc3%rqfPg%=P36SIw07`zCuasq=sEe)?`%$6h6GX)7ocBZmjJdg zED*#cTiO`3we<4H?9$&n+v`fx7S{Cj(@u487krsGblXqvhg+CTQ8tDX32S%|c>jm@rmuG9uk>a|379F81Icx;8{5gw;c0 zX&~Nd9l4RQ-wk|{$Bbzynac;_nZsyviJ9iMHMc(-5@ydRS`1ekf@oox4~-I*cLD%J zF;GG9eGP$ZXlS&NFUmi5;|$?~w=z{+ep;Qz|M-+k-G5AOnUaNbUbcRg(UuU(!R{6C zvA!Eg)+OMT(s<9hSYgl7yFIAZbrF(sRS@OPiWXAt#?P!$DS5>=pq`E*4LUbmBvYHS`;mHJZ z!?haB_oWlq!;z>?D|jlO<(YP8n}t>AStQ*xhMNPL-PzF|{M@Qt(#UTJ>%!y-2!GXL z{pp{KiyZ^E>@30(rMxk~w$TSx#C>KOLo1xc(h?3EN`bXNhx$l|v^TdTbcsjRs?g}g zGzaAq{DPZ@+5qD{(|TU9`>eOon^FQMgOmOG(HeJjLgBPoa-*C(g0`D8tDn{PEI<{x zOs8hIvsm5=Nt61K2+4HHPlgTC_Y8ey;dB*#es!#J9en=9ajD))qTSa$9{^y%?T7I^ z;dZ6nw)1+aPHmX-tl3IU?ZA)|`dZFIf-)L?<<8NP$Rr)ii$r(&V0a z>b!@8>78&msVwFiL=EB+c=C;Dl?5r7;SdqdI$;0NH_=&_*YAXFR>Aa8ZQyjR^Pgan zn_hl_`6oRg|D-3%e@jooW|se7;r+kzQq+ItrDRXbdM#4uA|#~0s_U`51d0?O;rw`s z7`X_7rTSX&0xQg`la|a7`%)$sRiBs}P14{D^aL)QHzHxvtATKTqOpUYGw-vX@7=Z< zUf*+gz5#79i-VtO;e}zl7D&6|+r^+$Xo*~&l_V=tXW{tiBh@%%=}LA1iTPbgmured z%~FT`!-bGz7zJ;l9ChYmhw8&^m^6FCp+2*aD#(`BoB%SDt)b;Z;SKy}V>*Ai{*U<^O^;wHKmlM5R#wp1{i|L*5>I^Jbo%Kl+)Wcyp97WC92~NVht&fZWBE96-w1=V#l<1T*JIsC2P&^NkfR9XU&QM%aFJUl#8XABt;2I zQleeCD9Tb;+$lB~`CC!`TON}?AvT}$P|QZ;;i6+q9w@CtOP{4mm^p(QH6ofcNiezR z`3Ca$%gf;|n%y$K$gl>Rq-KCDp-3X`eIu_IjHf6l3HvKx58x=U$j*y*-g!}Ba?uoO zGnIn05$mWgs<}zG|8b%*>LoM!f;9zgscnMgnp4LUwwdWG@{d4n9EPzqy99`1D%%tp z=BB>x#r26e=+ST-pg7qWTMj zLkgFT_vc9?5HS_mIkNky}ANWFr?q$76ZJ_E~xjfFe$rWlOf7_&tP}y4q&(z z;bIH6v*~=M`Oo|VuVrUF{{#NVf51QKzvZ8Q)I?SPh4(Eu>?|@1q47W<9s=gpp!-6$ z)<_Y`1!;c6lzDEqR4i0ks7(E4>JFvoY(xO97X2U&Om*4|Pb@w3g}uLZKI%-1^Z9uD zLHsb9kVchiqD_^Wy|*etrVPxA!5viAjoo@m*Rg3cozba+lY z&~8{al=82@j0bj%x+^#R4-=TT`;3pmySc}YLiqI5^AjAT`CP$AjDsf~xpv=q-Ck%H zPN9a}@Tl}Vx3O;+(dRigG8++YbI;sQ(t@N)Vk~DQnlKFHWNg$-2q^b_v&LlM)f4%J zJ{bm)@}1e>Wy#rmHOGNT)d#z$9^Tp4yd{ivLJQ)>xz207SKGi@YMII`*v5ITsAUd{ zHp$)v;#R4t4RyXkm@w(g7^;S<-QR{#kj+KKx%yLVY0&}(QD2et3o3vL^@hh}9unDP zj_(*3@qpMUtM)L|yT8;tGYVVFYX% zqpDbCG0ia>W2-BEjf5xw>5qZ(-OK zu#v;h$3Lk5_3B4vUcvM6ynGG#g0!_z2PqR$hsZ$BS+Ls}Y~loLCoBRmZN^$b5LjY# z-|?FvurZp76_H<91VtIFfqpuOw#ER{(#~F58B#wvK~L_-N|6l%BL6+pmBO6#)D@sw z+DUCHbJ^|}suSfd2zJhJME;9^Fpc?3ynpu3VFoo2y@N>bMUk1F+8g)UOF(qZZ~UIf zHzIG;uDxh=TOP}yWpG*vg zYcCJ4w>U-!2^Vr8GfL4s^F)1SO3Yqd@SJV^V<%*GjNDcszAlDEJw2r^MBy<<-=Hj) z-*(Tez>qq|8WBe1uw{oJktWYhXL4ATf66{MKw7}+AmnhXswLR4^xEdQl>BbB>3Ukk z|G9T-oEx(2Cv~J{5iJW7zAUypL77PlnO&zcOXDF(5vPrTW8@U0J)sJ+?0g5jiNL{? z{9umvSKr>C9{ji};;J!?LV`-)>zuy`izbE2twxhEA>G56()BpF0P)m8SvbBzhW$`~ z2;%LeP1?wyQ+Nwe^V&~rqu$S zc}l+9a8%p9a-&{@0E;aM4dx1!$~${9v?tIu;|-PPIxp_t@2&t9q^nIuX`q_1xNH6V zXiowLXiwN}7Fueva}mjYi{IMAm>Ax=JLKM_yCMiz!@si!6d*poLD`V?aoyV1O3tI3 zwl~cMF)bJ+s>A{n@wgFI6PI%_!{$zAbjxB95w5JCv>7i?Y3)#6-Xz&@Z^v#r!ESj+ z_2plqvrwgFCgQc~40RG<@DnC2mlSM_>z9cakE1+OQcS1piLC2?I6vpEErDLiDKb0j zqWe+%JlRHZq6ob<1F0M=@C7#&5b3y?xf&sc94D0Jq8WoXF&Xz-j+P$4B7H2SW!vwuOU6?J`cM$A@-o~W1m>kz=e?!q9{~EB&DD^I# zTU-wlP$6%8XHZd`Jn>XInQc8`G!1X_HD9zfLg04~HbJ;k^SFsAEH8gC+hPub+7v|S zx!3I9=Jn_j%Jx2;yOxr#608gp_-P2>;dt##%|_?l=n`IFFA5@nZ|X~PKS>au#Jn(U z9fz=m|7=dO`r=S{wW%b-ea79v-QxE3!kRxhO5L#r&?D}Q8jaTH8NZ_~im`n(!y15# z-UT40i)7eXb;TmW8j4``U#9iXk$rcF+;hp??nfiIFbbqw4BIcqB>R&5m0k3lQ~&XP z3&-f*CH@^ldq&qW02YJbg1e=k2}Jp-^U4!B3@7WiMjmtk)T5kcBJqi(I-mPONr@_jAlQ)yEAZxgf)J~cJ~-}Bn~<-`WfPR@6%BJ zZ}}XYJQe>DYge^8;>m#lRm9!`!MsNCuU;RJGZ%j2VFHH4+OXA08jdkHI1);N~uI3!lVI7}X-a(9RmW(9!c z?NCWeHt`2@HnU0x}-SCd1UG9D$|?>ztcM*c-Ur7QdaZ-+nN zjrm{SNE=57TbutE_p78PgZZN<`HdE16+B6oI0TawOWh*v>R|$D!rwu`A3=;5G0aq$e>gvhf$*UaqS&rPeU0wI<4rm) zmUnFIY%tOq)l~B55z=E!$`iwjn9Yb&D2S!-bNgOMEwR)Ac^M z(>`2A;s)i6&!q^PtG&B8CCH`RA2`R%^VIM`KMCeMMo=N<5r|8BVbZhXD4cTyw0g_n z+<1!IuC%Z63@Q9p%1??yG~v5rw-7^o4JV9P%dsM_+N)8=svJj+_$*yTd_|7lQQg00 zpsDMAWrQ6moySV)va(KdM?Yv=Hc*bZ+!gd0$+B!@rz;Si1<G4g^+_bASYaIJ8*J=xMz0dmuR$<BlP6WWEG`yL%bk7P!PgB+|yPKaf*p{!C06FEB?61(ENlX5o zgJ+(uMo=_1vf6>-z)fCY-8Y9E0=`XQ%l(G#+Bzpm{(PE3(23DoEx&*?L_e``_xhnf z@^-D6?R>P=36c3jD+;Yv1^sn6@w9cnYixQs(1DmnE20&72`v?nHlveE`u z5KAKWq%PrTz{d01;U9w8=tnJZAj5^bw^BwEvQB&Fag551aJj-3QP_C^E?FGlmWBu9 zoulwaPb=s}<+Enhkb$!x!G@~14lvR2SnyxjL*4;Mm)=j%p0n9C-9pPWrjU&}xdrb_ zxf8WMvc?@5c4@a{Y(KiHpM^wuUZ(Qk(+?C4bu)B{sxN3;st;&gwxr8-TxA1qNd+fg zD#JJPDi@*a2SCfut6jgq2lr!tFLVze0O}${Z(Y%xlpOyE8O=8 zS{4K(C&h`2Ce!eRO6a>EGBXWa_gufxc0QR4vH+ukwDX^Vy*fHdQB6bxyQJ}~W#_Nu zIpz~JgU23x#qVSZ%Ug$+<}3;jqCy`y6AI6oY@S0kccp5Kqg5Y`L zIOaO$x^~-oUw!8GIEL_Dr`_$NLQBe_Lp`h84g+#yJ@Lm5xwG}*UdD~h>A$cg7C=)klC}iyLTeVw$DS@j`i4f z+%rQ((e2iJP2TVp_KrRVSgsZeHaVkN8Gl{Lm57}V~BqgBGI!0a{>k?HO(=q#xykkYkY=k}7cL1XHV3%2g>5ct#0!HKjuIopC)yU@0tNl4)C zSc(V76op6NJ?xG{Hr|#uxQu;spFKsz?H9yg(|EnIwT67-0_jQ*^;y;Kp}x-2WSu#d z4Uq?qEJ^wiCh{uJe64?v$_sA4Rr2oj>FEf2D3((oeawhf&LL};G zcJ~*-RspQwf>vGb592f3A1_U{HYuCB!`ujB&<_pzC`-9yOPh*jibkI=tjS_vmrjGm z5*$@3W}78wIu;;SnK3U3Y*hn3vX0H<$7L?RVs+N*(R@yDhK~$4XC$}JtNt9HzLwUA zswa+)w%M+7rc4)AV8n=pUdV!e4yXj)f04GR`jpNcR=ARC)?j|pja@`$c3K|S*!APW zjP)QS%->eCjtfTmy=Gxv6mlGW8h4Y+bLoyTpM@M@wHpYvNMCUY8(nzWZ3%Hb;deeT z4k%&FD*%dVH^q=k{v0igpQSuBadfxJ;u16|+g``hmz(!HfbZc?1qm1-CM$ZLm3fFQtY^k%<G3ItAbmJF%q1Y>*l*C)m*#_R$NcxQu;Lxu`O2macXhO#Jgc zJs?i8ye5rHhe2$WldHYZR>j4V5r?sq+e+=-t-f7*Y<4Du3*%#jm6qqr=5~2S3QKNDlz)+$UIv{@c^)ioFWXd8;va1qFp6Vm41tt?BK_hTs^ET z3}u6);7&As6pI&b@7!Id4_VzU#$yXtfvf6Xa^)(qmo8UL9@2fG{+JsBXq&ypUGrRB!vKtW4N6XOZjg9F+tDJbKZ)WEM(;*b;lN>VfH zzf0YTIZ8a?)gZv&K@1#uS<`#;xy#O8`N?@fFlNB~k_s(bk{AhmA5f7t1>GN!+$WLT ztAeQG(+Z9z%_-wg3r&Sw(vyJZV*Tv1D?OA~|wOzJALcv$2NVlZyqX=&B z?`{#89g2zA>Og))%a1K3BH2 zJJELrh&xZm3q5mdSGm8TS0r zTzq^)m371!Y>!?Yc=-!^*yUxb^#}<@*i5vK{Juk>T78lkeTGOXP2CkKl$oe0)Vk~q z1F~;Rk=%ZE`Ro`;@SS?O_VPDk3;9^iUHTUVTNa%n^DRNr_>U)dOobSB1lnNF68rak4j|g^4B!H;E|a>kdlt&7W43T3y|d<+S_X5 zRRDdCjlOxwxQjc#M`gcKi}n*FNvDrl%siq7kEu5%ALZ3`g6n%yeSTMeMe}AVTeKA?bDDrC3e6*6n z-OJ*K@4ye59;^_GGu({R-$2seNEG+dq{4Jh0P>{zeRISl_p4q1K+@ zb24AtNuN2wM;!Kj7p)*HnS};@5OjxzyVXik3U{jduMbMCjMXVnj+3sW%F)KCWh>Tt zr~8j`MfgXdfPk8W*5nhh5O$rR>qJG?Bt_O}pzBbe>r_}fdZIV%b)UN5))0I_5BQMW zzK4EG?>%puet0g(2VQ8}K@`2)Y5rDS1Ww+{)>u6tbyDmMFFlgPSiC7JEddm(Icu+14Dh7lsE%UdW@RcUac(Xj=*KRZY(j6_tm%o{nX z9+9dV>ES-gHN&YK$!e5BgaCJjqpm|8)CbdsAELtVGr6U8mJQp~hjIxMaOLInCD_B` z6zeJ(lrw6kCT|I6hM2XM9&=0ltdv`v+g~Zkma=!YDZj)8Eh?|J#p*wcBQTMxUR~wN zmSvt+N`5MQ!WT4=uwyc6c^f6F;(?W3JiCIA=qR#YJdCetNnIaiW}>ee%^aRoXy2ro z&wzq;nQ(16a=OhT9l?<|A-7N1&qZ9#B$vkh2asd$X?+kyEMkxfOl389krQ+Y9IZDO6p!9^i zLM?|RwegkjhA5?E3q5<%VsjL}h|7wG7)6bkmivXYwuaJ%%(8};imAPtHcL0B)vxk# zX92Ay)HC8uCtN1Z-kFiG zF5530I;zhYL&MCHq0s6I)XJLSK*P&>8ajhRYo8p;G0>$i)?76jnGqc!$vlIP%i$VW zpvj0$Kd|)>TVup%`M^2DBl%)jfayeyTcFCQ4ai``quK&Ws@uRZh=3EPWlQz=uAPK$ z6P3@xkc==~pXiF59^Z<=@B+M1wrr}67aQzK@N2q+ZaJ%q7O2nAy0W@IV1^)le|z)m z#d07hU9e!YCC?-rQUXUOwe3>2`Xp6ASOmfh+l`Vz@#~!t%ERYf-sRLH;b9vp94U`n zjLA%Xp_GA?fvu|3pG!VZH(Q?^BRNlDK?HzOH?BUM(tiJtI1f3jdNC(YvgxmBn+yji zyntCC2E&MXSf!{QJDc$V_*j2pocl(oIBYn6EC*4%eB=R0m|48qGO23V9{Ezf<`>|9 zo+WZmIRxL2-n$JP0D$%1&QihPhY9%sIfm+Hw#H_9*8hMxNlKdzh)PJ`RxFdmRu62EN^z8^$Bc(=e1t^qz5&|Kyq!heg2PUyrQdJ= z`o;Zn6nWkC4v_88V90NZeJMHJdc5T@^_k`1?Q(zKpX2@e2ER*rHmLCVMaRW|+$Xc6 ze2M-e6p-Lz2Y#BbCLlv8<;s-wDd5o72xff6KMOyE|$mX(5usV z7%;nN5&M)_QK{P{c{zgsZ-ftD$8`>#YW=^k7HEU*KvbAC`}6&YpvPS$`qy1W`rBP) z`kgvzc7UL9_ksf_CqZ&ZGg{dSNcrHIV4clgz;GL01p7-~BzvWxbM_&jb@nL0LBDyB z(at0gdf05lz^c;ngh$*nPdK2Z^wz+(EM4s{1S7D>Qcw;guSr%Cy~gMq7|p(2DVb_g zj$j{`vjN@6RoJM#!%TTGWf;UM(4`fBY8%_q-lAYdS@FXrcspzcf>iD?MKha|JNYo} zU%d_Ef-}+T2=mdIc}Es4rIcxSI!hY)l1>epW4i{_63|{A! z+bs}(%%%ORG8YwH`W5cbB8a>wya%O5@tgQ#NI{?Icbe!@>lrL;* zHAbRyYpg3CWD-l9d~>2_KC5xy)bVukF-KEIF^DyYFJa#~lnNkDLF0~S#iT(XZsq#; z)U6Pja7AntJ?B1Mg)}Elqgzk&6Qg|)7i>*j;HZ>b)9<0fp)zYEG%J_x%K&Q$m7&y7 z(qh`ve0tf(pa=Su8iP4hcA#Bqc)>%i<61y#^3Eu3tjvlux?fS&`(Sr+owTZGkRb?n_~LyC>VA=3^r&6*G9tU<&}qWe_8;)kp4IN7+3O9L zWq9S6vP#cHs?+hGJ2!IX1LJ@c;KWQc4QvxMS5kK1=ZBd8FezyefC8G*$6l6%m+kqm zL1x&MxdDrt?kqrn!pUv)F;=wTU;O=v?wYRWwb&#eoIKKceVq?1;fG{JW$+6%?<ayD!thd z?&f?foO72V-7L>b#l0xCkSbu>VXW%Jq06?>4?>iS9vr^fl6y*W+Ua0U>TvquZ_|}L zqMSUucJ(Kh?v0!SzLHf*U&5i$`>gv*uBiw&vTH3Fk_ey1aCq!to5$-{Z73*2Fl|Clr9YT)Jla6!!x8dHOXbW{ zIHyMJcz%bf_>MVEHQ7nOlbe?SH49=(Qjb+|pqWSCAxyvIYf_!XNw&qJ(>zM?F{@@v z+wL0l_(DV%8GPdqcDRJK2EFBSDMKxSz;;tk=^;k+IGj#X;9)iG!+}ImAF+VtWVL;; z;wr*N-EyhzahP1$yjYuMhOpPXEoCb#tihE;#UQN?^JQ?CO-G}Z*S4k70|x2_9$$1< zuZ~CI6#ZF@cXk|Uk%oYcY+9*&od0!eVHRFGP3%f(|ESE^yi~yz{QjvY9%;9AXgUsQ zqS8{-l8k6&FZ8ZSiw{<}w%eeT3dBk+*YE)>mD*q{qTtl{A7qod3E7up?+VfV#Bh!g zqS}0YWEUkQJDEY+sZv&|6s7~TVvhLi@Z@F7wdnDRbE#3Q)xF;A(Tq7gRaJ(8#*Csb zX08+yBQv#kA>|eQ=rt0$;D($kb*kk$^6ZV{!n1esV%jo&zyzp@bpGn*Ctlr9TcY=iomgjn=1t{@Vq2Sj_w48M z97zf0Hz&kRpusbA!>Nz_3<{cPHxP_Sv<`bxX%8J*6a2KwFJqo{>gLSlu{5$gPHD9J zzDv%LhW6EC50*%UNuY#p?NHz@u%W2J$@@O9gR0O%PyExb8)taX57kBkc`^$_XPf!u zsbVX`;kq}lv%mwKpmQU)_K(}P(Sm4x{BzsFKmh<4|Mj+67`Yf3O6oZpIXM2e0?6ur z7C_$CbIut~5nkX`_$x@gM?!6I6{R4+oHsN_GX9Y0iEm0hUu1FQWa(PhDeAOfavkejt?fuVK-E)fO#up+wkX z8w)pOJw5h9?#U#v6pC5Jtz;_L!yA?E$PLw&hvM6IlWrwe?_-@!opdC$Egw~j>UD*m z8$IoLGK{u6nMN94Abx2NkO0QwfL`~~Qs5n$wU$l=o0X)`$(4+zCXG9aMg@dg5tm!W zIo)GQS=?xaE!ACefBheljz!R)o-yKU7lU=;qnHH>8G@MHbkd>CmM8rUmPhHf5-|bw z<9b$-ii;ad@%B$W;E(gK)eokS!6LpO4@frIe??yI3#*0Y^~lbqEIyvsBPR-gj;|s& zb7W7&#%&0hmem)RbAy}Dn)468g4_Fuuk;Ic^dB^caX=vZV#K}hH79~W0LN| zl)P;8z;_t$083+#?pFtU{O;;W`Q6n=k9`~Q_~oYfMD9P^eYBhUD=a*?+z%ts4|##9 z5~H@3nD$dA!g^i1ebHW1XE9klv)}%L0#j*mFNoJ$du`r3{6@n{r>JF-%!J5@FfePn6(rb~X+I0s7SH{OqRMKrtsE;5|^@imD0Y>6f8+ zYQD=haHo>N13~Y+d7_ep0}n7o=+N?L{YdnzC;fWR$Po@+@+N;r<|jxU%p3d3cUOd; z$V@#p)Rr3>VP>ISW*@KE8>yJTGjUx0JFO4_4;EJeL~;8CYjo5pdWMSu-_qY7q|`kWjKW{69S=f?c~d`suO6Ck3&^ zN|~(2q{IVXezB|D8~$=zeX?$ZUmmC-Z4dELIXvl6$G8SBB9Whwahjk7*pXhSh&-S7 zps2VpNIt4VLer^lbn3Vy5G6q2b=|QGw!$Cijl?oK z-LV^#gOKTuRD|ONA?;}2P9-0HJENMu27%goqx~tRFRR+W8BCe77;skuxOx%ecD}T` zfw#arwm$#ir)Y8jfqVNiF>CVkO#kb|tdg~vv5ozIQ#>C_dd~$y!uJAMyN$>w;N$GtL8(bXy5h&X1=BPD zW<#VJ!2KBJ6%;F>BLqP^X(*XTTa$uxo*AO&3w7$3OHVGL{g!DhIo}R8S5D0TA)JXg z1jy=?1ghATmV0KrnM3g!)n_GxZ~0f(6rC5G#^s&I7UTJqCzY|smABH~K*zTxGNP4d zF9w6N>wL^wBe7NPySK z=AQecbH`vN$M(?*yv?};=S4nrT&P7$84scoT2|)bKt4rkIyMx0|zb=pp6_=!Y1PA!oP>%<;kjEWB}?`aQf5@cPSC^#P`*Y+mqXIa0R~7B=rN!@K36@tx}S zn2&!bG4VhL?<{_9kJyjWH|1@OgYuzavut5O6*}hfqnjBwqz0r3fheOi+j2;>zCMuW=R~47uCSw^BVbx5_$?ML z6VNp5y5@mBwT#DV+i{Y3j>Xbj{1p$td)?SZfFf`jMy&D3DhAuSEtW{*-?Xu7?gbJV zvZF`8V$}*Rav#55dG{QCzWh{fHfs(d0mOOp>%3xH_ns+>)7Jp5@jk48O1}_SfOgvx zSlO|TK@*u2_e#08TR+rZj~)lh0uvN<$L{5Vm>7c$-ou^wdsJ;26gbiK=)ot^;b<7J ze|$nS!Vd(pK}Q=255#76hc;VhI5T#As>*DO3+u?qcmoy#tyrg@8)+9j_??y9%h^n* zZjYN7T;M?pgH#z3Es-G&y0L!8;nn|q0{b2{Hh0t7ixoiKEP*}yG@%}(&G%LFiC`*I z?PHaZCz$KA?X@A{r-u2W&>x(yE^%lb5sO&j~Xf7=rxD3EjZ^ z5FuWR1V7-b*SJ5mSpGK^3ERl;!A2uMjGP5>y9*lMzHtte?o zp)cD&(&EZ0%ekbI8UC*4h>qa2Px7CwvnfMVRH>iv1^5YH&VQ>Kf5O*Q$kxWdRME`p zr*PD>vi(1`BO%>?O%2EYcqhog4S$DdH&H<$SS+{eox0K$H(>Zqn#SiQFBW zpNH03xiWk5zYFtJMl^`r1$qhdq@OAqd()+*TfNVD&C1G3yO>zM&XEDw=nVj@-Z95v zC%!+>QeZ90E8J;|HzQ&YhazeaKVqD)Xz8uhU8&g=bv)}Z!MY)^9*8?f=YPY80UVBr zEM9kV_GHmVTdANwzyY5o@|=iOSKTlYO)*;6Ln~|TG8o)UMr3pwFUXipN7`6F^s9JT zK8(e-vlAVBlhmR|vP1{%TeV+&M@Vv4I2D_fbLtOu@sPpV26rV4AX>y#Du3xq$UDDY z&MGBX$FMeUDcWaFc7!qz3<)#BrcgCXmCZ@Td486XSgSj+Y>i&72KIw;d~-r-Dn6+G z@lk(H+!&_sZ;REsPjx9|j1gL)fA(qC%bv}z>jU`Rc{VS!y03Q<2sKz3A$;Bg|3I_+ zMZ(c<$diB(mlio9DV)f)r(G#7O+$ILI0fZBLThTJ3f(Smt(w8yzreH4gx;u?5rku+>4c$teZ`nqU%@tBJz;Cm$Q4F zix&flCh&VdwHIsEt#Yo~&{T=JR;X_^}E1MlQ?4t_!K|OU!7WLaN{4~rq)LyMEU1z%GwYb zqv=QP5Y133T0!tYV4+-?RT$yhCXIK3Pj5GW*6R9})pRkFru{U-^u+{c_v_@v#kJ4( z9ffa-T?tN+jty{)o?tIuBzfo_e_$aJ4eI&#t&dfa$LKsYwjS@vube{8-F+zo=B)p5bNZaKsHd17wKA~GLMchIkLg&$3BZ1u8v<&`Z48b|!p}CEG zf~JG|2~8WObZ&_J!pr<0t#QDti)F6|D#th+`H`)l5Nh;&yrLbN9dcHs2QrD$z{^Sq z133bvQLK{QPI06Lxg=ESh#qgBA4DSjJQDikg$w;4mJxSjZbXO0N^;;PazW!cnzC`O zX=+1XT&>C6nE6sAnK4I;6-T&YJ&X$-jkHr~t?qriiXA3Q0Og3S5)xg_9;Nh;*3Arj zQ)b>1*wZ7s3_}Ky!(Fj&iwaZOKNAN73zs7$87`vu`e$aTXepvzAu%f-xx{p^?C1>4 zZEdPj?>0i3OX#>5pg96W<-in8HQlHoBQVSF)^3(o%3Di7otuU-omoV`?5yu$;Je_? zVo|~9+T~jLNpG3Dtej)JTY~@G80_V7V`V4tk+WN5-vJ^0RIB33=c0rSqo`Fq1mj90 z-n_-4^Ru>F!DPOACbY=9#B+h6D(`R169cPoygGdm zjW|n;t+&Gm?KOd*pYPNF#u?%z1sG9uC2w)=t`ce3Y0fF25puaI9v)u_@jGK!~Zu0)SOT&PfSoT2JqPwf~i!1jZPQ0~!CD*kx<~~&( zke-WMtQz>#bKs-jezT%1KSDU1*uWIi4k$wH-w4lswGR-VX=)gRM51#{;p5ZUoPt1F zR5kf^=fAR2xX5?q;GW3=#W5W@PO6R-eQ6M5W)e+FDlzcBadPfkK{y0IUK)~6?4RWH?`MT5w%9wsaUGHgZ40@gABX9H*c!u)a#)&v*Ec?lf8S?a! z#a!Ujz!GdmO-4;VtV@t%1bN@mz|T*dbUjK}Jm8k}jfuGK4%-i6cC>j$()OD5m-s1f zt5`d|LA}g==eE^<$}3^hs4UXGnD7L1Zb;+c8MHReVyeJDD-SpuxLH(>E2g+VLiW8N z4}PbYeh1rnifnrD@NUbebrk`4(Vw*og!k?5Sf;fZ9Q23F>Y_dPb(X=w;&ipS@Y?fjOZl>5oJIzoNCl$Yvo?=AqsPGrRFltADUABcX4hlV*zE*-C__kQMtaCJ{*W) z>IC-kW%%o!KI=AQ|K#KZHu3D<+oQgf7n|R8``M%W*<Sui}^Ha#BiIEb@9(WmABP&9YqvA+utO5T}peJYGnoPioH@{*I(j6HdEaa=BYlDsc z4#P`jDGvPlG`O4Mh=H6Zj?T<&iysIpLa*Jf4450eKaKEBVyFzJ>+fj{R~Q|7TfpfM zmQQbl$8lFaU2P2>`{%Diq@s0B(IIQ|H#lfM>J-%UbLr)0|wbs2*A#P)Z zJUrf^B$oUZyfu!`pfo!Qkn3Cx425xLiWG;KaM+^hwY2!EOjA(Cz-11Q;dRmBva4t> zLye8fmuou+XB#3O!<4tGJS-+EF6*JO%Lei2N-v`sOgprf`s~PzYfj^+7VG0WNw-pm z73vkDaR-SkP;)oS9L36laXy}H+4A-y2ij{Sqz2(6=L*vd(3%RqRL}+OAHtc`Ln~1n z6X(GKv684dR@J6z2Q`#7JE07BS6LvfHaM}T&i+t5dkDraiLqq5fj-1VU5IB@v=`^m zeWTB|O4U-iQCl&)0e8WILD)w*7`hYN=$DnnB=y_lzYCZb%$xBmM%58WU zbUyk}C@)Otr*HIiDr?lWc`6G>iU)->wM7ae)W!AMe0zb*``Q6c27ywg>xR^u97%(8 z+L1u^K8iBoJDZ!>S2EkWSVgQ_!V>)Q=-MjssA4A%rsU656Y3ohwAAP+ zsv(favff?@Yr%eHn2MeBum!R9aExOn3thB-ba@!KChe_3hn_whtl3G+X;_MUJ z{FVl(O1Mz-!Ro4(5S>2%bCmH#K3L<%MXMUi=|Kb=hYf1SfsNHyqj1(wvObHy*kn;k&y=I;jV+l_SdC#urz#3fAN0h3(P z3M>ikXW?IUfL0CW?aL1K9nrf8pi`4F4#RwtPFHekP6?5Kk#b-Q-#;Z^6FP^V5qmi~2TE;D^6E-tiAAgwixs~^{Q9rkCG61EyKtJQJo_DjcBi@MBrvGZduUaaFWt;Qz!F7l8L?`#z=9Zx(l@ zLKnWn?=sY8$+xSE-11$>>MW49fmf2XynD6a^o_bKijtr-=jj3VuDYU3Lz8tB+FLH% zLS>_H#yQJFkREKE#}ds%WBxe~&GbP%xMIJfMkxyJSQo)DoOQZW9>Rj9l3oc$hSlOV z1>9Ywj;?jB z;ii;l_-<~o*Vk?ffB7nlRMvwzC73WEA#hJj;|g~BfV9BF8AjmjCG!kVxQDXJF+$81 z0H0>~Mpb;GP@)h%UyWI6z^yXjIWwV==OH4aFLipfbUC0)9lCL-qGLh2%&|{UFV)lhjjxKMJDo#$m-!7p&zK68sVdGd|or{V0>?v zDNd*&%9+4z4+&+6pw79!HhMU~vNA&rF>cX(|H0=QEk=~}7qWD_|0i<8H&e~4P3#|; z-DdK}s0r<3hI{y6y-*-zj6}jbCQ>neNb;XgK|4G~K76NFy=^%c)L&&4zLf=b(SPY{ z2zyCPf{4|sibQ%(+&%Z%-s(q`S!5)xVPrPOsBBjuc_k%zVTNS>r$jeFM0&4x`8GR3 zm6p~oh$^C7l<_&yDBac;iNti8s*{5ttBlCp(~Bm+$>Ob;|Q`v>98!qb}C3fTQbX;&(d* zNm91gat*$~(?tmjHlWYiQ{hF}hdpGYK|?e!Dq$0`RyPeIjf$T-=b&-oNFr?iqZ*vu zqdCf*}&GaUbbZ*uJO$!GKN(~E&?SmD# z()D}v?Y;=u!Y04|oqUPw$iH3#!K~Azf

Hdx>wtVa_ zXBb`?i9R$#q8p4ecm7Zt?&hi_d z+sYTvKN1?`;@ubO-~GoUG7u2a|2-*++1eTj=^Lq-TDkvwYMtMVe+=hQ@I>wSwb zh%^Y2J3tJX2t`1U5cz%wDW#`}BsPS0YeKtk#OUf_3YXx)9H+AJTC!5LGGT2_Qwp3F ztWe!js&sK}tvjc@d!?~W)%Oyi>4DpoOcP6&Jg^r;_G8n>J@4ytSM#A>asTjV z(<2MFnyU5t$pT30oulA)vk@~%?Bj)`cc;(cCw8ljs5i9l;{`4Uw&JBy_NM6v(}mZ= zelrQw^~ZLcbQ?;3IG*q4^KLsZ&!6kp8z-RYop)fK@Z*Dnd!G*=`VtN#0ph5#m)x*; z5azv19p2fctI2n6HZiU5B^^=xC4}?XhAga*0c+|+tQ87G3wCBst?89c4=!6sWx0(( z7b#1*2CbfSmbo{1*`7w*cE9d?YhO#x_ais$oZK9C+v2&{pj3$i+VPQ<*oIV&gzyW- zhl9VWV{Cb-=F{sLV+9v7AXjZ~y1<2-(sLG~EQyRRM?c=-%C)ZLaF8ofXXZQG65|}~ zew+(vmFuonJ^H{6QLl&iHRJPL!sDca^!H{ z$=C(!FOMn8K6e6cxc#Mw%;a+4_P{0XG8JvTLmh^-aMP?gPLfS}QvMLW9GuZfs%Ss` z7MQE?>?X9C(hq@A(uax2Uu}+O^3T{%<`x#hd_~~nE`2{6B7)3GQB3#@@AQxU#%3I-P;Vlu20)ORUYt?DASX#W78MBsiOaSl}dK~~+877KRlSF?8HSBkeNaO-wq5-R<2 zx6E+oHM)aR*gjFotREos-}&CV$!>qx&DG@I+0aZ`G6u_~d$IFc#0D*~b%#R$j+l#f zc(!J4C>Y8glKr9`g?r1`TSK-i9mKSs=-q~-SSQou?`n~Y!|s-8Y1nG)Xz4{m?P-#K z$c_&0>%EXKqYCM1Mavh$ecgl6nd`qqTUqwNWrQ z&Ncij&(NL&L)R@1U@X*wGkg{t=TRKl*p)c*Eb;dWIzJm-1uy9Rz6=E$dH-)Nf zpLGR8d5$gry9NbOMAgQpK?s9GHJsU(n_)d;+MIg4T1j;xks6DMP&`R&ae{n)!PM`? zWK@`7!c;}&xNvxG;-jIySt^)6TzaReX8N9`b31w2?&5Mo*jUqnV|WfF(m0vvF+@@> zx%uSLWuQE2P&_dvsbk8E&(#Hcd#c8Ov5}b5)&6{eibyLotaC%>#(ovs{(pHwaK0$G}aMI zMIL>78aG}786y??Sf;sUW@bsn1Qe<{hLGzYzAN6*Ho$4_gRq>WTn zelVgZh7?xyvo~U@je#o52j1tI>ku_bK8=!8R8n8EzVIM6Zn<5yRI!7cVV|aKDqnL4 zH9*!C-q|yQa?H_mF=D3)d)j>hWn#9Oy#u0BGBo1%r-!YCGF}|$qVE2!bC<>S7}Yi; zI!{^kbF{(k+i&NZwd!gFmEf|#AqsT+3qg256v_w%I=|x~zTzvgd6$8(Xsw>o?GhHU z0}zH?=#@^N@$vGd{6ciAmD=nhWT;+bD(v`A2s6UKx(NN&KtYXhyNdm@O_CcQ+UsW1zRFNlLcdvApA;jej}`DmR7_FBgX8(fUhvK-Vsf2>w34oqXP zULUcWq3YZq!nDx6;7=&jj^jdm!zPz(lo35KQmyC``uT-@%RJ>(gYqbP(C{>-Tl903 z2D1n%2c?D|tk-VuTm#!%^}!nJlcqG;0uJj{BXX+BZ}-TWdGPP$415d|>r4X0etB%w zAhKpDEDQtC?9axwrxOem;a>H(Wbi#O19k7Vg}FU(@z`g_nD8#phsFX|4~o<4>8?r- z$5bs!dWn8XyveYL`e;Y&=te;W`%|?HY3Ff!2Vm@N!?yH*t_&B>Wf~@sK;mbbbA%^W zC-i`~fQT!0?gF*d*Hk%Wdv(;7XL29)ldm{(cFNt&v}A4x&$Kpg+dEp(DWp-K{BhI< zhp?*&F-?hc#pZFp;#MRwqv9z=&I-mY8;YS6Jr6z^H!?DGG@n9?fm76iFhNQkD3uPh zN~w?L9KjB2D>9{$^4WlguH;B2rM}9$l(U~`(_fc!czU62%na1wxmkZb&oWwPoqB?4 zaOEE#!NX!4HOUJ)&Gb^+~WpIvcL{$;lBH)ZS*jssGlX@ zFIxP+U$p!uR$>x1cFs=UyO+lL)+j_wbj4zNF(65>#Jc*%GiI$;sl$>r zrP{i5ac;j(Wn}*SydP%=Qfes{9B$-Ti-{+s56sm^HKZC+kv5Z82-$7RAXnLtXCy_e z)QIv^KZw6v0G}&7K9Z{c-To)9zW%)WCCk`m+HQGs1Jx|E5qG_tekPq8qd7$nMINsW zCrLnT)TQEZ^Rn)+GVKtXc{EOo(bBTWblf>b`J-&It5tDulkOs2WtsLu?d;HGt<`&* z>MhCitxAm2VzGi=%ac#^@NOCmnYv5Ez1HX|D1tFKy^aiMg3$xLH*Z8gGpojgOY6^T zXISI@dKI&9IM|-C>Y~(=Nnm=B8TD0_S}nSj#Ktp%3`(5581g(9vf1*izbvkfnVP-Q z_ZmI=314w>&rY^gt!jPK1a|bERx#wLdvIM=nlnH5myy0W*%freN46SBYPX-MtNTFX z_S{sLco(`X4scM@<3SpaR;9-k_31-Ig)F;m?TSlN$}r+})|3|3VGd~we;e|IPOl~~ zJy;u-l6GRKGDmTQAtbS9Pza+(Pze%$z+LPs{x zJo1w`=(o@v3968r*5t-WpXh*3fw>p3Kg4%Ow91D-k`D~>wC#-$A-jG$wmHI2uwpPo znFwzA6#4ul-qQuBj1qk(rVSH#1ho%nXKyQv2IQ$D+$0B7So{DNgNpiklRK}}p+GHb(2Qsbx~bcvjxhrVL5Bs46uK>Rsl)nhUN^pq!OEZemE)U#ykIBG zIUN0WC;IMT>Iy={KqeahX)4n*yY)Gh+tGBp^Xr|~&yu1XQ$)4b8jaCNZujRE#fw6( z4%%gk$}E8MG&MOZBLEyNbEHgtfS1xI&Y%>u?PoGK!>a$g;-pk9(O?-WIOP&5kNl0i zV65Qk9>A2F6=1aG(&XGv7mjAt>`=zNvp zfQN;$6?MvVa0k#?wDcNodQ96*dMMlZcJnW*;Zeh=ezqD$iR#j(shHw|udejk(GBSk zVK7mu@ITrZ(efCydz#@swzY;5OJM)WtX)ah!8VX(x|USFgCx98eX1`m+FogAsa0%N z)R@U>t`V1RTMeCt zAhVcv$qi6vIV5?_W=)TAXMxoZ(?4-c7VU55kbAV&6#(>_I3r{-4T@6YS*i~M05C66 z`g!KRwwb9tWUcJG_bNI2?ky~pfuZwuOySbq@^&eUSa{<1IDK**Iu6kE%-sC-SnbSx z{No^mEt(IOj=DZv7&G{|;)TrBFr9+tX$P(Gs;U(#3;T9EiO&^GMV}S3?6L(59|mfD zj9T20*S{52{Ly)rZF_F7SV>Bg`}l#4lHVmr*VYz=fttyLr-2P_lVSuzla?p+?Auc4 z5~~7(5o{~p;FdGGmt)@!dJ&*vV`Kw>^{6TLhgIJv>f8+c{Q|(dOR+-qcs0E9m`@?# z-t?8Ob+r0C6_Zv#|#3L5%&)}357eSqn z70dH2g=NoAKAA`CLl3|yNPKgoB_4jK)D>5BFE_I-KzGY7VCgoa0)Wc^{v@P=P(-E8rCP{>(w?rRE5f zpqf$1vx|(=MldVv-wAFQg7W+WDQ+Ftpq`^sh;|zdsmo82k`vb?WLFU7FNc^x6fiGh z2Y@sQkxq)+_{0aZ-8m8iI~@##MKDjpm@3BZa8k_Q zW|Z@%oETyrIO`E#iVekz{QjRT?qKKWKhwTZ0`nUsg#RZf@&5n_WoxDNUx+?aY$1@~ zkPQiQGstOdLmd=wv~X+(aMEFx453OCr{fVc@Q97Hzmno^P~LL+ioMSg6n0iHWgaeJ5V= z7PeNiAh6J!pNMJ>L)TfTU-o`y;5_wQ^s-B8D+$ZOZI=;N>+AFb00DkYrmJ1&xG>TO zI=0&ln>y~l!CCKT>U>LERhWnMS^JE@2D*A4!~GGARXosZgIwsjREE2WZ}gdb&3Nmj zhgo6#eNkUTx8qewnOjL?)QhnEKRc8XaRduVI>}W_Co+3GEVT}4#~l~iYEy3zGSYfB zJ1$ix49|Oqb}yB_q8aO6-02ZAw_hxc&}1opuLJX4TWoS!6e!WR`J0Zbh|vzw zNiJ77=9hTZ zQMS1j$>wC{nDQpRfrg%P(RBvND)5Wa$Ns`iZuUip)Z6D6J3{c1#Xva`i<2c|{DV68 zle`2gT6KCOnyZGmrBh(c0m?N&Y1@=XDnISitZE0wWFH-OdiEokuu|FRBVq}*TLW1>PJ$9$lHHx@6TMf<6b$8? z90nslcl!bfXqK?^2b(7;VX7Aokoxl9`PpwyI`TP7kg_ki`<{5+oljk1sICwajQZAB z-YN$@o=~5wMa(s4_wF2m=1q%d6U<2S0rQUyg29pFwT|b^8f#q{(oKe+>f@) zd=L;2h7dk35H>ClcrFmSzaW(WmgQ;iTuqN=y3YG6PGv9;bbX|Bsa|h}vZsMcEzy*TOps~%b5#l%*owT^k_z0*n;Xy`*hK^QpQf88x(xsAOgq})* zb`I3s$f!noYI?C+dSYaHxmub|c5Yf!u`C-}r965P=obWV(B`LvM(HSN1Z=?X+zBu5 zdVuZi?q+HOzib5iH+Y-T!a2R5o^V`mg`C!hhHACLjXCIG~{3E+x%{D5H*xwwgE2 z{J{J4Wu#948kgae7e~q>wtlG;J*VmN$TM;qst9q2_A;W(35TQl(GOJT{^y$z;Zarp{ zHCS}19%i%Uatm!{%-Ck`H`)3GC;2V7+FU$PU&{qpb=2mA)|S<#qshE8W^cW(us868=0b z5z5LzJ0*!l3==YF2*XivQkkVSC_iZp`NV`0IlT;bKe_eTB=ZPU=OwVpSl&l3-#bd&APd!sYbpKeu?}$fJo@z|8Mq#?EJmO`9GuitM(UZ8pxvwl)%A;#t=+n>l0^@^GIX1c=pjefzx zQoC*e2{YRwLX@v!{|oE%opG%yktNAeF=j)Wv12xnl^p_ znz2hbX=A~yE|R&FS89f?pWHF2afrMW4TI`CdFyNXlBnf#iE^uE>J_c(sA};gXr7gU zM*kN>PpWh2mP3dAhD0=bu0teWRvoK3Tl9p9iv1t&9sJuT5dLovmHMV){zpfG>E8y> zgi9M_K~%stWde8`T9_G(Pp%HF@}-4k`7JeaPSQxQzQlVz|5=Un*phMc>K|;nzVn;r zG)zNj`W~$t-3ZeeFl-icmR0v@9#fO)j11oTZ67aAh#J7VLV-PD0G4*d%thS5ICPcx zh>FbR8`Cq?_U6glr?7%LlaSJfUHh4xZn>0SDVt8u`PM z%L&6Vl&*l1-6vipD@_M)8=gOjpgseXPNR(0%gq^o%TAm(ngTKcjM19yv<7LWO4R+U z_TayRN)M^qCNsS|kFdS#>&>Csb?$hUm1?P)D*>q^q>?%~q-M0r*M|=RQKY&QiE%ty5oCdtdymGsIA|~w}Be#PR(EG*>rU%O? z!oe%_hk>P!A(yESQr|5|q3@-7^BPGAOk%)ePvMl99}ie5j)dJj#y z4w!_ZbB1BGnL~<8lvsio)LE>TQ~8U4mGa^0b@$Vq3l5S#^h;G(w^_bc=4@n<@;}%pnt%p|L{fG-Ol*mj;8YW9)Tb#uW<9GwFU(~xH)Bx zdM`L2i1Z z)p5FYbrqk#+v~>-A{IRf&&_N%I2H;M-Ya5{Azl(H>?_4<(6zcM;z5c$;$Dg}!A~c3!P>!=FEJztN_JjY1p}AhVC$z>Ez=*91E*0jy_k7={0y^@@pKm z4lA;kH0fZ(h3h%)e&GDPJo zE!N%aMwd^{aCuuJ&ax)l{F=-_)x0BtTJ_1Ib*KKFstf<REwtYWoce&73E`cBsZ_(S?dgT@9%}0vEuLM>HZF60CsN#}lC+l>6Kv1L0bz*(`2E zfbAIZyS&3zS<{H*t-B2qJ@X#pnZ|y#-CkFKh*j>HT>u}P6YXFPxGyJ?5J$`!cAaCW zmVTgEJQ2C0B~H3m@y-gSVvh-g-kOe{Vl0>XoB%b%Y!dNif-!f4aL%fHs!hzwd7U@A~*3)rOLjzJt@hEGmfGS{eNx-Y`>PT((aDl^68S zoF>@W`(z(@T@4KTuSgu^dVX#MN;_C9XA*5K;OsZG~GY(*&+{Pn#>Q!iXOa;_h)&>^3 z$tgmc4uh*br;y!)+-!s%o6N17M_p)-P!xZ7sR^8!4}4aLrVXdBptA+J?R9pSt??!-%1Hog4-j3H zQ57YWGHce)udK^4fq8^58bfIS=MyG2OqZ|p&|J(_Y&KW=h>f?7T9&GRo!5}J$BFIUj z1ke9O3vZ1(zZSs(0afAsr^Y+~WkdF_0GFu_<@IyX{fqxnlUXy;sy2yq9n#Z!ys09w zf$zrO+Gu<(VA+h67%DJF8Mk_9E{%8KqQaWAfHQ~m$AMLR49Xmt%{gu;4D(9QK>DK#}=k|Nlmum#R@8N|o5L>7jnpa9fG$l9! z*ozN?AC8@BZz~YhErP=#J{k|I0NT{pX#d@9h}TDan6K(BJ+K$vSJ~by5MF<`%C3o* zARQl_n4a=2>FZ}*SoO#=3RPCZb#>WGnwPImFFUFiv)>^{`5xu#XOh>4C(!s!L)e%6 zZFbzMZ|RFI+E@NwDi!SRDo`|E@s2O3S^qtB7p{Ov*O%O&tFjlnADo+D@3XQOzu%<8 z?hjzA_-p*K7X_g8+kGIPfNZK44K%Ksdm#Mon6NJ-<3Z(eJt@4$7zh0M=74kz~!_*0Y zfN7IPF-_DYrw=1CdIEr@*eiRD)>hJGA~B1IEF>auJm_q(&4~@+WFv4ur1oZ~{LkO= zZ_TcEZgyrqPi{UxyZ+98_7eL4p8htW!IL;mogMKrCS%U$2D50ZOU)=WtpN3)l7pbu zP3*OEeJGnc_kaLD>+m$_44nN0&Ey4%JRB)h8ZWZHy1r;LaohH98P`QihO9{SmWxfL z3II{zw1{DH3}B zkti+t)b4}_32!?5quxH>jiM?VhQ-r`ye?H_i=`Fy)FIgOaRbx!&TCP?fhAvyhn9hD z*pQpvX{mzi$rv^|jhoBuQc>$deTa#PcxVLn)-rgWxwhVW>X%iTKew*x@gey&OY{jP zFcM11&m`3%e+-(0-R3FX~SitakcD~Bma>~$Je&i=J)`}eq` zJ0Cq)fu8L|+}sctk2tJ+_G~`0l)$&%1VTdzNe2P;c!w**V@?k`Ojep|OpwaSb2^Az z)&y>Qn7;=MRU4@jB$e=EhLNPBRa~@{x_;o*QW%#yISg{t2DwCMX6r!_Hq6;IMf<|P zUGM(@rRxk~LFO}H>AY2EJ$3X{b|%bicT6|s8n6>HF>zcr7hd5*jEvcGxZIU z(8m_h(#uNfk37g5ESsmji9{g(JRjHc4t_GaPmR@nQ18QRj$4h0Pz?prk)=?SEN%#e zBJ(ys2HDSpfF(McXXl%&@&Tcp`k=2_(WUv5Z5nTay{|E!*I3aJ8MO@m)TfJ+kjQzg z-Cu4+#1mYyFN#0~ zq`=60l~NlAhcT6~;-aLqI2DYgR-J~hfVY85h6XL!A`c05^FUY|LgS-@y#GoZW&FV6 zvOz_Ozby1H4lkxQlUe#mN+yV8(3HL^razH%ctDZTP+877E#IB8;#|SNt?2pbAf8Eu z+2<}*i)l4Kbq1EUdSO7aRuV;qgD|osjo{|OOs=i5m`EyKV*~|Kg5BuvkNTyQ9%JK_ zb}{#U>{?ZnNrTLuqE^%N&ZxGX}3 z0P}|6HjNo!DD)LXDNln5Oml?G(dZ)=iCZnS*`iPvB&3z6U{z1J>%Ml@TD+@~AIP&$w(Wg~8z?zWNOC9ngAuqA>g!-toZjwo@ z28z~zg^bmpIo|*q+_NH!g~y;}LG)?S@{NnTReKD0xw_#R%LciN#h&DBZir6W3N2Tv zAZx9dlX;$&2#XhT_S_9;#&2(E#xkj*6K=N^adnLNUbgRnjP6A?6}>KL)aLjG3B-4c zkT)@Y{E(~890Vghd*{z%aAc8;vZ|!?bR<2J6o&YHP>gKQyfY!TUS2QPNl9~8>{QQ7aA4qziYeiJxmUJ z)p@uOFQY|&7%59B7V`1QMy&iw37nBdPDIEhpxW-(8yk-pqq}am$&Cjy)m`lOj8C4u|2$)q(MCfEe(l7$W%f z)v6s;dtViQK1^WP@$RNpAH9W`b^*=-ramF}3e#S&Wmeprs>TRou9a3#Em@7r!<^mD zLm8dvozb{aGTa8)fmW28-Y5wT(AE9t60I_yzIX@8Pz=?FECUjr<2Wy{H*nfe{9HH1 zKh6KhxaChe;BE!6bHlpK6WmqrqEg`rUvjwTS8?E%8dUlfto^~OA)c| zV09ZM33!figNB*Tx#HNGp;G%fN0jc=!m559hix_YSlK&BXC5kZW&=m(ERnoc09V=j zE7>VkM$-j;95AKN2@VaeCT(C$?HjMo`ll$^x>xnbvV6@J+w0!w8TvL>1#i@*b+1YA zPBT>OcWQAk!!()n{)SMMHR%+&_4ocjn%TiUP^3#-RySTUa;Bvnao-EE)!?MlG{X~; z5~#@jq{s$go>j(38bO&2dEONf-Ig4QcgD)A7SYkYwtL)*TiTUdRMi)s&R@s3Gtv!0 zua@}9uXQVd2GorpZmWo~*dV}8CjHKI$Ud~gX@9okK|OCL$l@_&h+JA2ZJMUx99du^b-ogIokJ<`j%66Ab@bG(`L7t8VVc>7EFfi0^q&g=T%HtvVFe|Paz zuC;hpNvhC$sAC8!`8nq*18P!2o(%h3)^31IChwh{z4QFU;uEDdA3It4<{~Y#S~Gfz z6?K^NAXMvSs@0*uP557eF8}BmLHz#cRXWMw z1&j+7CrB~Z)l(7dZiq1t8BeIqkK$4JE}UzpKD9ToA!ATc^>@M-xw}f)y)m(w+7DQl zL_hQd_+e1_8v;_U@%JhBD{H%J>lof9NBVxP!y8`Ib~`P8_V79D`!YVIHXG!2{{1ag zadd>+rr^{nyxW7EJsJ2fD&%dkc+?kufltK=)zNL=YGrbsKT}(5fXl3XvlHGwSHGQ* z)fkhiZ1#~AK(9T9$##=l7oXYh{4DM^cGC97gwS1zwdZC5iItQ73ka@*3KGAB` zjHhB^+&m{okl?%04E#7DC>1ETOj6x2Y!{B=8~D)Hr_Sk9LsPraPX*)OvtT?lLvryJ zA~mQk4H^qSH>J3RrG24M%PJP_o_xf27B4W7oN!GvyP-gGNrY^d z3@WD1MNcS+(Nl>+e!j6%2(=jM=@PS(>3C#hX|kI`X)c{ew2$*##~bWuayy4w{yR>= zcB9nxVLD;aoio#2P}AL~`2iR_7G(#mG$49Op|#^=WTm29qzt3f9pd_vS|#M8&uPHd zTYvv@RVa`qHlYI_kZ@zafYIODj5^gxYaNBHsStVjN%|?K&^mCvSb*47EuVB>f(CLf z3f{yJ?*hYbkcZ4GVmJG7o;rSAM_iX)T%bGW*IA|e{qf}SSnM%9o{4^Dj%IUs<@4%O zu;~L<%+Hr@ge%yuDEd>9v*TR|*=#7Q9pIr5EH#*?m{zPViiE8L#*G5tkpzzLC{f??;xzO6h9Cc~& zRnrUG_t>a`>qYNZWI(qsQiZmrK2U1&$HuKj5~q(8H*3X~!uw?JxyP6V9-a9C0gX1r z{f`%DT|FSp>DzlfYRhQOZ1`z`e6wIVks3)Wf#8)8*;=T!m5B8L^bQ3pXPL&ZAqtKr zbex~{Cv=s|g$jbh%6wV8r|6jwFTTN=26Jq4Z+VSNNx7sS&$WLhx%01|ae70ENh;{3Yi+udka`dS>&&%b#8NO5IL=lV{5bFCKN zp#=XO-zsNg{2z#|f1UX%RW1K_!mlhH4GB&wYKdxir4Sqx%U&Y30A-=|8p*<(gSm=! z5-p94%l=gn_;s@0E;;|(;?#;$3|4Wr({~;@esMmZDTA)&;aMR zE!FG%G?4BfJFXjiGBHO~MSwP#pb=?Yay}=x8j*ZVV!Z3&N!iayF?+b>eD$6@t0dHo zkeP#`fM=qUUro5zPT(x;S{g7|=N<*ftNx1h% zZH2Q*RWQLU=_6+@jE$AnD=rpVu4J@u4QDk%LD zteTTc0(w$LE>i50S>{?*0g)aFRE1a)!3upW@eR zQBGjT7}D9A6n$kAF_V1n=rn;hesRPTTe{rKIbAH;W`oCWHfczen-CXG;s;ix zmlPT;5|KdyfO>qiUTVE@eGQqMSnCH%#dNCCr=yZ3rF|vwR6d{`4Z!Jj*$`{2! zwHHkS`6es5lurTP9Jj;+2%kyGD^0IEhDt-_)Ni-3k^iX)%ueomMu~u;=5s#_|xq)o}KT#^i5b6!7(;m1!5Oqez z+Hz{y*WXofc(yA+Y-WK-b+xP?QIh$N%iNbjvhSuIlbx1ll55gTIB80N*F^2Kc-Npup8; z=WIw20cU?+v%X2i*?`aDmVmUz;tqgy4RXR~A-jc4c`+#nF4MJ978f6jgLme0pTtZ3 zMf-W!5KtV-LyS}yV94#p7Igi!)_N?qDd!OtoP9jK_7mN{u<8PgRa(Iskx@Hu59^B{ z=?pvB4!-`tr_1GS@h?V;o<$_l4J0Et#+N{mpot%|`(E-5{5xc=EY1F9Q&czbKSJZy z`q84uHyK8Y{h!*e`~w;GkI*P?q?dvEG$>gww0>1npHO9aB^N}MEJ zy(e`Qo@=#Tdzzp6ni^>|8FzWnemXf{w|)mYE`90VmUiZ;l|XZ;I^`jX%ND&6g7`EW&) zx|t+E-`B$Ks@&z~>cSad@{;TS0;>i2D&PNd`shyc(SzfM+zWn*P5Y}3$FI;Y$NE7N z;{4Leb&Fx~BxSrd6ei#-dZmq~rqZqDZ$21=A z4vF?SQpI_F=4=zMz3KFV$$A#26{@a0B6>v%b^UbMs_e$T_Q_gR>IUIr+8ZFEL&TQcw}6&M|0*_i^b)g>{|1U2E%*@ROQ+%rvXOHa>OU>%nyyScTdc%sC0 zl+vJfrI`(m$FS8%)ejZr>V-2DqY%&r@gns1oWOeC^mG>IG%JHSS-G%qFJs zw|pvI4$So|2WhqI?~@J3k!l%}X;x%hkPRQV3tKDYAWHEpI@D^UJ5Ib4G&Np6;Krei6(dAqo|KktO;lFiXF$D8 z-TjCr$vWno!(nM6@s_NsLZC3E6J~XZ=L+MvV^PuRQ(%)8R#b~9$QUdyI91Ml*5`Ew zq4vzHhTQu_Ww(n9v)u1JQOX^}=;KkQDi7e)rEhUEZ7GsZ=mE#h_$YNvA}}nImvt7y zJYCHccfj-Zm`8&@q*_sacUIH-r8KK)Z#T|UK8Tvc6v38fuPURMQ|pCYV&jqS)X`@7 zpWYMr{VI?um?+p88*2So=$P>P)u@h`I22`!hg0Y}{6*R0mj-s0y%7TZsz%$wzNqEm z2k%;$GNxOp6$TBl5h_EQGsKOGRcDd$xvEPJA-7SnfACF>KgOIB2@X*Q!|C(Mx^)|f%Yv5BW@=5%r7w?y2``pcWe#v&P4>7I9sW+%qM$^*s%oNE(I z$qf}{MOa5Z#5b+1?|dI5Z6665RGMryC%vRv%;qpQ98YP zRbYi+Ua^`ZEbaI(X9W75z=8-ShFzp}a|GtiX6%86O9+KlrTBBW(|8|Y2&0*pPKy@7 z!Akut?nDdwxgPrr1|$5}TfUC%7(y(zgI}jCM%`hCb_|t%l7zwjP(0Bg1|z_y%Lin_ zdC-Z%xWgli7LTuP>nW2gSPt?wxxIfw7humS=c-GNLa`Q(sjM#nKjYRRRRJjuD={en z*=wshzjvT`M#-|S-mubWS_fm0uvm>?ZL`WsT6YE|HRjt);6UACF6Z&KCc)adm62*Q zskHi7?1K3v3O*!;YZb2WW7jo$Xm=~s+`|jHS0vG8yqn8Xucn*g47L9-lRhakdi>qq zt-V+}}RI3YnUqOAA4q`gsps0J@@sIKbZKev3zg_TPpID0wK0cqh zVFcH@+{S;13SNPFDf4>e-)T7foYR_$1fK`F~1Q{G&dvJulY zC|ey%I5nm;DV_+$x#Mx?%N0%a4Tj!7G-*kdAe`By^6wufugf+Kry8VZJFoNn$*r;nb+2udV>@ zV(jjhu|#1i86=|`&QW*L5mA*#e9M#aQZhR_UhrT@>!pdaz`@)UTr}sC{d{}Z_q#An zrCWN)MwLtAeH+gWIor-|vYl7j_H73yU4yW32cmsV>GM4GC3^~1X_6~b&hll-&10p; zp(z|IuB|GMpSs7!6o##tCPR0jzdjjER0hk&Zh@G-lzx|Xa~>B8m)K^!H=W83UbHy4 zy~XBFm4}dC!X}zmL;zpkYE7=zyb136NB=(G{$;X~N_284ND;S5jT&y6b)+ zXC29KZ(voo4$&p{e}rdpI@M#*t4m)SOYf7GH9L?*-#afTl+~U|j@sVKP{y7*Fi*9q z>k)0^1~XR~4sB?hP8GyMEyt&JV2vz%GI=Ke0eByz+YY*F4B8X+SPDO1E*%1plm zz{9z~c8{VtKo~6VN#FnCI+d_zfNfsoN(XI3H@o6Ij!g*H2q4=KCH zbaq%lRn6h_nlP5wHTF}r^zvt)Y0cZ3U)u8h3Uv{g<;P~Dax0xCXDGFt;otrTN8(@h zU{YA&$@BBx>V5VYR5SiuPE2#{weUJ2Iq4amE!$F`e3IJ1Bvqyr@Am8vH!iSyoh#vD zcA!(S=lPg@#rDKqQGYjnkg_VC+acuR=M?y>?dVS7>HoUwr{*+FxTX3COl~i}wV;=> zc#3v|W3I=Jccax6qt=r zJyKtd<=*Pk&(c4CdP;x)7?6#lRP}2|^a#kk>|3Bm$WL(Fqb#Zrit*e4&tu|vvokd$ zZ)n2D#W-P+L(1TzBXM}zTsZB?fM+rrMB(QF0xf371H@^^5tI5}t5N za8R5ngK!5M4Dyd&Z@kM(U^yZ^WF!^lb!7>wXDOjm^WSH#2Q~9*E@ylDNa&OGDySbp zw!c`1i672f$Vf*YiZQoX)>aHMt?K?-)eu`%d9LWjTU8COsLRzgDi2f@E$|+EsCE2- za+(eF?bqro;?fJ*bN}GRGlHd7N4L$_PEGC+rRerU!`|`%W;f?}6jNR17Ke|y&HU(q zxt?}+A{^93oEkLh6+MbsiCE@fW$2w@kRYjB+FmuQ*t_8oDi|5;=F`2zT zU9Ec0o6&4n%NVRn>X_NA8*q+Q`|}OWoInmcgKRT~{4$*k?pT6dHvZ20Ri1F+QTKFX z@YV+Kh6i}#t1!AOmj`&$-|Lt6nA5BG{rz!+=aPiahU)+i`P!(*|W1(`u z`BPrs;x9-4f){pWafe;w+;DoDY5E-mRR=z+8}a%ViRx|1DW7^KyQpd4TM-2)dd>w% z(i~8v^oMnpZ8%-oVvF~~&%BBL$-k4ift(vmHM^mc{&$mT&S=j5%?5{nwy?;{L|V8+ zQ@e94o@W~P#ND28s)L2W>x{le(CD5F$^BhQtMJlXZMa*?gnG6E24pBCMru3@_YjxU zl>qCDUB6?EAa585aZ|!deM0MlXLK*WpVz%E02$S0Rgl{(fL%?=T}aEkY!mC}{40d;^h_ z5rrmJn>R%nWm*+rP15!OvXRdic-RNPp-*_({QP0#TseRovxeN=HRE{mMqg-5sY8ulu;!%# z<0a%;jW7Zs?wa<&gM0p-uqY=Yu36!V*nOh<*K2m{(v1QfzE(#@gq-{C+l$^mO=qHD z8**^pcIrCc6B_4#qs{uK>CD{G$=38cZ06raGgVtVWi_;~%?p$5^+~cGHmOj-|3%tA zM%Na8Tfg9rz2juZJGO1xwr$(C?H${;ZQHh!72n)CZ&j=RdC$GoZmSP-wYI*GXO3s~ z(MSLN!1N}->l>Nl#h`*e^2|ZvYP%}B6(>Ga;JD*#=KChO) zT=%p8Fwi+;FT_XcBdNfS(4z{I-U~nrSmDRS<;NilXyFs%=fpWg9Tvu%flunGjF4mW zR39MX?9ASo8IAD9y!vwBMqUC=ijEb1*Sy6?SdCX&4hKh=xex`hWG>hU4-y5AgR++H zl0r(HxsV5}-Bd;#Il8GAPMkac?Qvr2D9yUyoOamieU}Ot09x(mM-SmRQ!$>jCZ??w z&3@n_38t&tY4Nm~(3HSWgr6XRoba$Ws`9 zLQ0WeQ56K8LcN$KAGT?{maVZn_Y+^%X1~Z-ujm=IpPzXwHD}*d4r1QQ z`oxPtDt^6i9x$yYf=>ukLR=* zuxHP`pRtb=9nE3PYWZ6uU}ycps6r8Ts1Al!72Wkj0X@ier`|X+J*JA`0`VTzD)NBFDz-p%73U|zupwKZJC@MN+^B33!xx*k_d@DQra2*= zF_f}RHdzwO_$c%;i;oHIFNGwuv?o=EgevIpq;ddSpL_1uuJv-(YgtpyMdi>P_;u`o zw3b}!>q74hh2|^N0&3y5!EcWPH!NPHd&9N9+1l2#^+JT9=jrF?U&hX1W!4=mCi^ak zynAo@!JNE;JLHA7Ua0~zrZmpSL-30Z41$D`V3l!|_W0t7%q!^*ZJ*D8R=r@ct zTIY!uetj1?igSqHAr>L15@Xk4@;|YQnm&$Ibk&Gp#^YC(mg(9$G1xE?jOph!_FT&k zKfd&dzcFZ5eOHy&oTE#3EB!0|40r@b#=l+0U$=mI z7VridyI{NvKM%J_X-zL>T($fqY$Xi4~O&Vc4Q|6tbQJ+b+jMCs#nhq!S0q5L*r#65h&4|zby=8+n~ z9sV10i{}*%xW09by?L~OeYG3C)Y%lhxS)^X_J98l5rML6c2+*Mb+>muFsOjmquQz(`F!B>e%UZ39ESxAC7}W09R7&L-{t6G)5Me0?=x6QIl4f{9w1Kc z$>RksC8wp(u|;-CW%om#&yn8azqhSfD?{f^z*CYgA20h zYkwRzIgJ1>D`&e!)?_a7_EPtab$8c%kosN`HsW*!JcZLz{nLJ?(i;G_Ig)x|PHkY0 zVJ?)V(=4e&Gp&V^t?xQc^KYo!C`lVJ1$g+RxO@F>aLd7@y%qc<*dCGuu&sxR?3qv|6N%v9RR35&kU+zxE_F>6OL7du?T*^OvV1Txq^9d`(vkDxAf<0_~(XJ zdL_yl-*n%2CP&I|C+JDR>@i4*0CpiHU9TI$O&7ZO_JHVtBxVWXp`wHjp41rKK_==n zB*WiOB*k5XO@7k-RwVf2H`M+eF){(>i?_cgUP?NDZx;>l8s4VHk$6p5^J4nVPIg?7 z!E&q(Z`f@@=nuA(7@(MZYMSi#{HFqHq^E*9`Q!0){n^jp`+wA974%(+ejXuxCw)2F zAA0qFwc2cj|8hVt*mN{kE0hCMP~Zt~Ff;sdqZ9DzOG`q*PkQujm(m;cN!wDF`cC9Y zTQ&3Ag}5z%bPXx?>0S)&;yl^r{^_EwG&H_hAZnr&0Sw0#c6X%_BW?z}0Z@@NDcVeb zC^Uo_76{&YdtLrWm=a79XK%n7d*{~=V}jg(iVs5U6Nv-yV&ttw$AC|KPw}@mxUS%){8GXkp zb0Qe``Ua@sTk7`r%R)`l9_&C+;gvU?=8W7vznA}56obJFdriK)l`zaMYuvO5xW<`d z&`UwMXC#c{!!=a&&sJUy2~E}UwTu)^C<|@D@|U!ygOn~!rZxAQEt5wu3vl-fd&~f4 zLiN5_Q9f?Se(wPu1e3buiu}wg_T{vQg=IRKHwPrg5o6Cuw(}sKjoMMgywMyBa5oM9 z9r^}8xKpV&kHJwMX#3J7 z=RrsO;{=xbamq{l|5Yi1w$^s~|6lnURZ7yf3w#LP7lZWn)|diNI|%E156ZA$P~jrM z38e}}Nh+XE-SmXN15fopr`&lp$5raVX$D;Tq(enO3hB31Bt@N=@R@+=F-_PeW zMn6xrQNqyr-*Bu>M)VbV`HDxyA)1)S%t{yzb1}#&*K}&y8+dB}XJmfW55q_kq;G%? z0c1}&uW2v*Y2X9Hw@YX?<$!&TVZgvy%E~$tf8$vfGUTlKAiU)6A8o)%=Z!+q>XY@g z&0TN2yT+tMIxMk(tJ@%}_ekXWm)RR#3 z_=u#q8}<#{Sc9{vi5~y`QTr-EK~jZIsf?!_JcwpB=fdTA#9nLmMLyhifAU4x#ktv8 zBR9AEHgFR;gVT>#h={z~5_wq05%!_Dvhfz7tpgjXgeWfEKa!y1kJXlXY+x#6VyZvz zlssmzz;Y7dm%Lf{*^6j+y{A~YZN=hUrYrVyAdR%*EinTeu&km)aY==cuy*2;t`5*}9x|Vr(?t2-@C4qZlKRHp^RO znvG+jE=L)3Q*?bD#u2Zn~u-}vr0^|qx=gQ)>2C5mj9P5 zf(-cS+r%|dy8T&IFVX#>>jmegV<(ezyY0m1*GL2*$f{q3k`_0RycS_-wgaFZyE=Km znW7qgml9y{i*X@s4idfHe^4oikVqWhGj1Ij;qTdq*7J?4%Wjy&ESgTEU2*lxDj~%yvzyZ)4 zaYN=Nl)#PIYY3dixJf<0_~09YSN;$S;8nRz{FSm_0`vjP zt9n;i_#PagTMkI>Ioo&CVRL_3bX4X#v2DM?eBCJ9U->FN+51xZ7~XljPrQFgia7a{ zu6bTGcjqbcwv;P zm)8!bQWhcs)rY_do$?FRmNb9S+ zy4K$Ca`6kpjprSu5!|@^_6ed^qz*^b%$@$Npn0LC8Dtu&lU#`HhJtR1*GdvBVt3%@*T`hQa$Mr znxEO`fqC?2vNSF8J8ehT^p9W2lSyY(=%r;GblAd}uEtamj#2|Ql{&O?&GQip#o|nZ zr=ujSnCUj&hW&B;O$HtYT{-keTs>rfNu0YHSpLwX%I5kqKzS8CA+>WMvGFuXV)=3p zXRO$8QqJM;ZYF1J^k{;nbj>I|E60u;mn8?}H(F>NgKQYGS8cMYa%GKDB4`?DOw9$A zVRJm>`y*wDpLhfVqw~l%l_PX*__z7Cem7mHzq|CYNNE~I_7#S$VB%RCvjYF_yv-!} zZDO2@t-)wvKt%$EUv=4U7_+>><&w+K6^x~!Ea8^cm5996OLwG*1lV8kN1%;iIWz4997>Gw;lBH^Fu$U4 zFuw?sa}@GpYZ*~ZeNE+0;yHWBL%%KP3L_feHAsOOMWod-rP)p9$pmoQG)PsoEXD?9 zgBid)@7E=>k4$W#7I71*nvqWmV>V)5T*uGrVB}w(&Y?1Gd^HC+S7)fmmd=^d*d+DV zJZD@jrb9@BZq525aeyNQUMXX=Ehi2K8Ck)#3lr^25vAw2@H_ zkleSXF)^(>B1#tr5e+iMO5gD23B`48tn={F!ClMWqOKpuQ#c2};kfs+TW8w`3ZJbmw zF^y&{%t{Y7Zo09U5c^&H3(!~d{v;-i3DSl$}1v}N(wIbk1oC4&$`m_7 zkYc#Hg774c@@P}^u=4Yo-GA%C-Mp^z$$w(Jsr);SEANL}&x>ysn;*Jip(IiqGILYa z2nX$o>%dK2TU7toZZV0ItqN#12oWH37IshPXXw$8G!gikCga7Zf0kIozM^N|GBDoC zv=>&Er1!OHLwo=nHTmn>9;L;^=0gCRIvnq#iCA`KJ39o*vaY8 zY%=QTJ>C+I zmMw~g!Um$PUv!7_{#ul`>beQ7H)lbd2+7ul^q&s z$J2aZ*ouVyVEKt;l~+i|9bZ`Jqu^E~JL(woKIKg~?QHlr?_u<;4=ku`#0UC`qE)U> z7`g}w439QJrc`WJ*QMC$0e2E>_`}TY_-_n$ zuYAeY5x6v_hB!uVN!fZ)5Un6&ZXbcx9))&#B2WIg0cPUyHgImCKWQ8t{3y}vjP$DKjVfL{|*~<9NK10*10ej;#=Pog1`DVqmwkKj- zS8i7J9I~CFhGxz8TYfx;ZgosJ_<*1oq5UF%D< zhSzv+f@?HWy;wFpkbMr=@OQgdx8E-jEqDJ$0z)@JxnssUl9~YIB(>-;vy(*KQY2L) znqnFPsfkhJ4mUJUNk#BJTpk|z%GZ9RuBuCXnoKb z*E>+YvyP^>jr3y_hz!C7JOblI4((f*{h`Lw%|~yUmrl&CF>a>dAJ};Mu3jiWTrmfC zD42IlGX0Fg#w`T4eRm(#pZOd3G9GsE~8hA}`@^5mK&0u93Rm&}n31FC2Pudq7 zxgVNTco#2|zx4%y@pzbXWv73bppG-#J2&hvlilw-tUJK9INwovqz*?n5WdN+5IJF; zVaNm|LQ)ZFNOx&rvf*x8gC)#fA_Jy`0PLKw-3K>?B-ygG7(G;(aRZV7+TBPB8DY8D z65&s&pmS4Uu97`TWdK8W{+>HVPx)SBNX^(XjKQk`s!l%!qA8{O+XZ6Qzy!m4NQiAY zASFbVAnag+q8qu-Bx~`uJZfj@mV2(Jc<;Ub%Rl00i7q~%yz5=l6I`L{J@HczI={?t z7>Y2S{&G$z_eA5_tRdi>Wn|ht*%8?~@BZV~&B@#p=XB3BBuky42xB`awsb?&r?;88 zi;^0?mExRgRh3eE4r7$JK%_25TMw>Jp`K4ulAANQOWA_XFo=Am^lzL&ZNU|^vB6eC zuyGWD^l%@qpUUxaS#^4k-ispVVmBvb6eBUlAsg6E8`5*xPKwjnxa@AeZmBnkCrChU zj;7j5ie-d+E7Rnl0h^dNGOG;s=KMPs5`|n)*sSgEvS`0hY#wXp?uXUP z$X`J)x8$7@&$DeE1tDL-@3v1uZnNkPUT}Qadf>4IBVpuvSqz8}m133mC?}MAE8PC@ z67M4;Yq*w^Q|uR_F4OK3=YRM{jE}%;*yJEtIBwOIqVVZ|Dm{BTYET~_hGDK7 zF51o#a9BZM<#L>tY|J#Lv^3(tVYL*yM6D)QomJ0GG-ctS*7v-MzV8v{IuqL~;rNN! z7tIWFX*@kem?Jl+ImJy6dYM51Z&-&Rg;uNe`hhb~$`uKV*JGOLh-DNDQSIfM2G{c(++4LT}YP zSFY?VmR!$arx3C@2+3qj4A2{^1>isr5w)Tcdn&Te5LHnI#s=9fTN^XX>zPl^h`Lu+ zbge##h*Udna(YEQZElV(mmsOd?x?tW@8b4#kN`4H30))y0osqtQr@8anZ~-$_iZe% zutLxg79j22VC9A))uB&UiVl1{f4>a$2cSpeW)g|Hs5+XH9jtulm}baD2&r7C z)0LOVcz(Gs8+vViS6k^&Bt|Or878>6F}Zm))jbnm_j-JJt&i!tPSk?G?DKT*Xlo_E zUc5^#Pmr&5{q?m{TJ&99@r_FGURY@XGxXG3L5#}oYh_*arBfpfbXKkP8O578;|#VL z$F1Jx|AQYBcvl=Lx6e$LDcs7IG$PngijWsEyPw=pnxrmdqHLdWN>F}~nLm*~0i$LQ zMQMr{J)^%gE4(NPF}tL=~UpNaF2*YN#$1uGsLoq8Zq^5m^UnjGf+;3lveYtKm#_8yk$qU5{NFO- zF2`It%G_BcZI$PMv;OqBY!7VH9_;nrPr`F)n!Ss%U1hf5XWXC4>ejF&uu69}9dw=E zcs;mRwHp!vx_LUoN)s!25tUecraeTH_rG&E`n`;TA#;Qxv6}nbU=h(BjpTd$gDM082Fv4gTz@!C%lX>{*oOj~5th z!xM~^msFNp!#H-eUcA0ESnvEDrVZeH*1WcUP>sz1gXD=%Hyhk4-P%-f+Iyh~uPT-{On= zUxdhE6W8;jQ-YJ|judN55@p(u542buXLq^7s=nD^8Qie%ussUc9+I?}* z7ra~^QPBEFY~PGT9cizurN}04kVFn>sB^;)j$!b^uam4rr4>jb1dKBEfV3m^8GXi` zs}!?*zbfE(K_@I>#Y(>pJhn(jB_Z@BcQqHaaKKru`As8-JevA#C_RE(8D7`hW0_ z|4TrRmXnqo_z^=F8l$V`mI2-fVAkRebP%x-=#ilcS9wjl_QAaFkko-{$t#qThPvY+~Nsrad|BiXxAt_laj%6u9rh~`K4M{~7fbNiPVB5N!6 z+kD$+n5Z+_18&r1n%LH8z|-tHbX|egPOkxQu+j$$;1Y$_r9$hYdYy{pt2_TG3o=Zj zxYYe<)C52IY|Q^ZWI-iYTXCEJv<>{%1uR+VLgt5~`6Z*-e$@_4t=iY|&$x{W>_!U( z&MXm_o|x!MYaLl_VavK%v&gn6JV^){uQ!xo`jQ6B9RI$+aeA}cc6!pu_xtq@rk7w@ z>Cu2C`XbFxzm{W$V8(#IFCQ8#R^7e7rG%X8;Gr!59a;yP=o4gOZ3bo%Wun@eX$oY8 zIpgRpM_;mG`?io8r^m*M)8Ro$mt#!x-$eb9+E=+>Hl+^(Z>GqS({psfO}ytBMVe}O z_ByIqr!%F(zZx@khYI~0aods2iUeX!rwJ41l}D;&Lc@UbDwsoB`3J$%MkKf(KAkERbobN=lbaYtab{T1U2AIG29@^^eIFQ(0QXxO#VFrFzg zlK-c0h~JPyo`_w6v-#agEft?*&tl2V0RB`YBLGT+H+&=qeEz1dzU(#Od=9O)lj>8Z zzfrt`@@G=o*|Jc>0ZT{SMvbd-jwUeefCx^V%JNjOCj>Uqesj?d!j+k;7u5Ng>pn9? zxk@bdL4`?*f2B4;bx~M(!15G$81;pp@B5n@^$#R;y;ec(mQ6&jTxt)r?B5)i=fTRQ zLPf;1E3{?(h#WC)saG4be(pE4$9V{X5FZ<~t+zH^r)R%q2oNF&@d`PifUdVk`XC<1 zpfn!OAVAAs*DxcNTUohnA|)=OM$tbck+{eDpapz(9scwJ!h?S-9f!vx3?u%mj1F`+ ze&N)ep&^whS?s2XLcjVMu)$g>7TUwKK#AP;61Gl3(hS%gP(RLMoIId3r&9b%fpqx+ zxwC_0n+uwj8oq#-X%OB!^C=dIha5j}J%aQ15T3rE^T{e~fHuJ`GWcVEn!y*mwUA|G zvfLq$j~EuFRRlZ1SUh7w6~iu}HP%>f8^t}~?=JLzgvsi0gk`+`w^&Da)NG*LpJ`t9 z&lw>7-_L-8zKxOX{}6H>tfXm+C5-Yl+_Bv>MY0w`R8*C%iCnva03l$D&sL^MQ@t6v zPLi(#T$9~oa7$c|BXs{ufFKOQjc-3bG$K{3laHg4oQBVDh53~7Qie_m#ajvtp3tq$1u&bp{QW*e6JM~Ma@t%#)Xz7Kx)X=JgP(?>`>+Zx~0f;kmtAwq^G1-PX7m z(Fmyb$(+wG(r2f;%=fcaac`>`9qwhgb3)JO#X8(#T&&xb$na6Wj^V4`mlB?w6T~Af zNuJ|cF>5&SP5(dwok13~Tk6Y1=R$#n_{mNQvwl=i zYQ*}MR{0gMxdoq?gu1F@q^A?D-X6zFF!$aH8SwS!M*Q_m$$4^iGt zg(ntZnotf!v~9=&{b@pe+iuh_+2FE6D;|O^;W5kd_K4_Y9~2h{&org-NKXSXvv^IY zV7v5QHO-DRZ-jj{+0S%gYUOoV(lTwv6SyHO; zs3m!QwX)_1LleJiOt$@^MsiG;dBxck$(nyY{@9Gee4aW@_{c7Wf|yDI1#J(niYn?< z+5yy7>o3}YAzvbNyPas%%J#comNu1!37Gc}7ayCEZHOm*pztJhBF^tq1Z3@fAmlOZ zpIyt4(vD1l(lO5(kBp#Slth6ivf;tw{sf8ni6o0Gi|u0Rhb|4`*5SnS6;cC51`*+E z_;|DoQpPid_w*PZn@V#MhWS9NMFWDu6%3Q$-wHe)%h*+SP1@~q=BGmoh|nQ9=P|=6 z6iF@ZbDL+a!CD-;G{lfiE2!2xrRlK@aJNRCFPlpZp#GAR>hRzb^>|AYyQK zOxea9$t$!azVoLn4+`(XR)*EDaV`BigkV}@b_f@V;c1|C{1ZnAettqdjACaFML7QW zg!YBSj)=HX-N%>Lr)3k-OC{U%_9@)fGIhcQ6yMYs)0#HnF0 zH^T*9?Sgqmsa`^D*K#X{gPp79Uwfwo^uCj*myB}UVghNATyeRDoA*-#zr;Id6-`xS zRg^ChbjBdNvoKex101BkKOuDgk|j(368!;=NZlvG7}Xmc90M)7()h&lR@L6kL>@jhYUfe?5OQFW|VRfA3{Z)5YUvfR}Lm#MMdSN$boDw z-Oq?n?n_a6EUyLARI%3{QtacPcvDgWy(m9bm50-p=Kp6@nE~BchH5w!h8{id;I-^r zIe+gL@Ybn5#;&r~beRCGd)^#isL1Zg+--wm#o3)xVPtOAx&$2^81BT8tRj~xWsEkW z=r~3XCLNHQmXj_aMVgQf$~@&bHrSOuNESS_HfcxFM@2Yb7>URDY&R%(uiiy;f6OW0R_CXc_us7;rmb+H?-&qsQM3o&-x zoxmoeUc#ilJ%c$rDxP{Aca^<>&6X`Cj@!7GS`8H}&v-$4T&Ahf6??Hl1T*T&D)oH5 zNwOZSK%vz5sl`>QR@Ha}sSf2n6uUuzx*ZyGb$m0?ynu?)5yQ=trwUWJn71BTo1(+u zc+)@T9oE!T)dha0!|3-0b=q$rwd6(Q^t9)5z3NluC}1K$eca)_qnp0)f{{SZNX4qbXVF{sl^Nm03N;zT zdtFAH937P>5)D;sNubt}oW6t^vbS5cN3fp=`s0->14#(g4tDqC{n!

2_F(t}m75Ujc%5 z|Bie}vDH(pta=%4Vbfru@L~&f?NTYj?#1u{G5nBO5k!WOkwNB^@esJoDpdW+mMGxH z?K+;*iPd>A%N2bmR{jYeI%MZF2jT&UXXazc<=4613UluCeTPusfg8X5nqlOc$^U&a z8}{%gU!+pg!W&+9!U15 zXSN{(N30cli7_$XI|L{i3&P^>E>STh{`@lLZCU^a?hWEL*7to3;7arEHTib``<%J9 zrp=m;ZgF#20|AJa;KzYwNA2=?Al+rFd6W3IyyEmU;Uucg!^X*bf_6A5wdm6D z-udVw!{Kj_hH&`XOL{s~bI}$57K-{dlrFEjk^lSR)KkddqYFXc5}rwFrB;|j{A&c13i}m(92l48Y(Yu#BM^uD%6_ykuahB$y zB|%!~q`*mBq(Yv(%#*5+ndEd8l{77JmgS-&fnMk&-$7L5LhfuPPtM#EpGo;w>rfK> zx&Z@zHX<~vKK}E=lXcp}glDWYX23}SkckZo^aXZL_Op{#ET?pDM?^&!Jcrnn(~39u{JmU+r>QT zVkx@unP>1i+3?AKqtQkI8IssM5Wf(?Y@zWt!iWbE2|kL1M23L*LNN+iQRC5^f`(Y$ zEYUfuX^o1dJ2G(3SF3LptE1ms;?PX0h#|^qyq=_^r5TFJbdTaDr*4Ebccdq0E zc*=csgi^-a6v$H}M0WI+l)+0OM0V`9gu#m=%=Fe2|939p;~K&j_6*b`I)fKW(99mK zF`zT+O_kxJ33dhxkooJ2B}k-GF1%*|o#O5VkPoZNaC?NHa)4(9kQjRIXc6p2_(IfW zzLf>?q7Fjq)Z5J!REs4w2B3p{V+wTqjRv&Kb>k1f0C_Fc1j|=L?LhDD00Ug(zg>QP zBlfxq0AxQ>sk~JmzjWdvy8rb19=rp8@=3ktBKU;iVE}aPS0x!bvHK|zX2?L3^0O!i zg_B{1(B*viDS*cjxc-XK{S#nN1y3?Br4Ur6RDvI2EW0Z_;P`CD!c%N5|4#GnzfQA5 z9*c1!!O0yp$!^@3OH3E*=t|$1POvtT<~Z40kV+;@<{pPz~5Ai1iODmG-u;ar#R z+(e`nXg^Y7wH%3k!HF2w(kxs;#H7cTCArU}CRfPkjdvtRiXs+SXr7Qp;XtFdRrhj< z-8M6$x>zOL+91=W&uWb25D&J~VKOff8#FFzigYh;;yfGiVLVAW$eGQd_y)j)gy3T9Q(Z{ zni%6ohTM`EdTAwK7klbBg4Ae#WFR`y*uy2tNQMTMDBmjIbMk#Ylv6zc@;+ak$gNLm z_*)v;RU@^Wo}H&={4RGKo|i^QXG(hRo3{op{<9nn%;(@Pyh5trS}! z_d2?yxl2(zKQ2eb7$42qft>K1>@M0EIYlyjr-EU_N*^MRJhGc6hHp+lEhiF^c}s%a z1% z%ce~!mQ+*rbYGQTW165RgzMEQ&h?tFo!+Pt=c0__)2Ib75*vhJhev2ZNB)5PyeGY& z|LWeV-M3G~C0pajpgnlZdN52>2RApp;uRVlpPieTuG7j@xL(v+8_+SR zwd_TiOOB##&-W=kDGd+woXm5X&Q>0mQCP6=m)aCglXkb3mp!Sjt!ELB^6J5X6$7+& z(W1{%esyhgYnSA3^6s>2{%cyR(i5;KA?*|MbXh@8(Vz--`(WQ|OfoH&Q(phDL8iy> zrPZYSL{MT(>b&!wkiD)IL2`XV;v^OcW%6hPDFY6~=!HUmouZMpO2DKf7CKeB4hqb+ z@!5!k2&VN>Z;bzEu#nmVhmNGz`szIQ*w*}uA(H5u)RI@J$C>9DDoW1ZzCkZ}f(e2i z+`})}TMZ6l(M{ z;2I+{clF9dq563`nCgt-cGm6bNN0MJ(hRw+jOSxiPKDbr+qM9g($$E@gZbOAs$%6V}Z|GN=X^ zs&-*9IRdqsr{@hBTL}zaO=}Gli;P_`Q+5(7D!4dbwCIt8+agnZ76Niz78rM@0HxM` z9MpL-6Q#+)oOQsav;8mX5Zf`)ZlvvZG@E{&r?^bb~m3&w} z*(i!ucbz8%sXl>BX!;)4OnDADj*va#qcaNzt$}fUjPSQgp*g3+9v^63s^V~d74keI zL(m|$4`UFWQ6hljTC`yhu@WHEym7f1hYAp@Y5KaSW|BR9KggfX96>CEgg4 z;OcX5U@whq;Yhv;l$_!7YD~&SImT$_lB?-BW9ZhDf3LHf4=FukXO-1M$||Q(%HAZ7 zA8*8%b+9Kyc*c*f!bxg%j;zPobu@{-KTo%^|zOjmS4? z!&qT52N~5z{d2Wm;42}l2}XBW7YYljDAz>G$~$tl!Jr#bmn4)Zoi9}9nCDoPZ?{a< zq|}>5=u+xx%W`EyeKok%Bt5@YEEcp@zAaW(J$+nP&sZya{uE=kcCKG*Znvcv9|U7a!frO{KTvjC?)Rl*#p zM=mi+@iX4AXRF4%ezr-evuPnu$CALSNvo^08*IjrQZp_Nk zDKqS`*$by@&j{u16ee|+m>u5kI>Y-W%9H*&H|qpUm;fZ%R|eNFXw{zaC)gI1<@~!_n?(TAxBHLfJ z)2)}2x>+il%SX#?er-(3Y|bZ^IvGL5P^xrkX=9w^ZbJ+!E*RANKh?Vc?Tjz(;rYm@ zwVQ_ek6)_$p_MT;vsI3(gTJ=&$7?l{3v^%Y#lY-^g6$&rYhud-D$41{uv@jMhj7Y& zyOP7)C{En{b=;>PK0Elp`(pDAMfbQ}2ws1s|DflD{zm_^a{`%sr*2Q#z##s4y-)0; zcG4fXctXg5ei)FFzfVl!_ll%ao{?Icr=1>x0K>mBpQ$|&$u2JAbJ#m>} zr|u&MEn-bqVzouh&02hKM2+b} z@YYIvZ`NRMia1!dj!oy9&a!TxIR(W>z)}0-zp;6-^h_XlfAqD^pO3NsL5%*v^8P5% z|2vkqNOi{%SsCTaCeb9(X=7eNgOcL!(lG=vWPqN5ITkd^n;u^Dt{JwrKX9uKmZ;R>ubM+4U^+)#3G4!w&b!`!u_!?fnAp zjt|5x7D|_I(^MveW%V`EkbZe1(K)qK&bpFKx|+{-f6EL zyIOqW36h@mP)`W4FCY6J*uPyF`82!Rc#~eixIjVds8XX$LF_@^*<74Y3hGkBTzhG2d&Bt9T$MH; z3?I`5TF9YDV5k<8`Me#=Q{G}`NYX{%MFf3KCFSAKC9zYvzPVsLbSbh} zp!}RVO{F0{1~i1F=g(d7dxJ=qS<62)6VGL-<`bEnlakKmZ`lM+|K5_B)W3M^Gm5y` z?DcgNoTSYsOD$b3ghrA1U3`yn#Y*Jr1)E_8I)3X{l?LI_UA~MESo8fsN2Cgrm`~~t zG@5@AB6U$ARow{f$LaVHc;kgJ&lipp;5bFol$*32HX?^UU2e>(*GrG47Kqz2;oJ`J zk}R3hI8x+he-48SjI0Ua#s{b>b)9mCH*-lkXj8HS#b*3$_k)e`mars zBu3b?&`DvA&$ac)%R(+G*Se1_`M zDAAVpz!R+O7;`Jlr;x_kG~VY z_Aaf+j_5u2!?H};A+tq@V>>qeW~^ym=Al@f^#~y=W1cj?8{m}IudAad?a=Gd2* zYFAjmEd#MpC#KlTXXsXdXH(cYP_zlfE?Q*QFv_|fG<+mVh@H)ZQ`+}$VR+Sfc%Zb4qcB3B9eK@HWs{=X6hD7;?ehhHAhvIFIDa{5k;?HhFDqe^E{TbXD=Yd z!`p#PfWBFNc3ty+Q8VNI=JN9O?`CV~_vKKF>}mg;*kv!HZuF^9HgR1kww;tC)WiYvsJ2xEjbO*GPeUB2%lRGGb=~MV)Gv1TNhl-QP0i1l(igfg!_wtSRU+POUCC2SqJAV>{Ej>cABj zDFXKWeei60vu+Y3H1arp3mMLHV^LS;QD(i5_Quk*27@UeIlG8(BuT$^EbK&`d1;ZC z&A(bB%FzZjlN$M1{hc5}QyR+nt< zC!z^)W)1WH>bi}=5nYVt5Cv%Ylt4f_lEt`JUXHT3P1fpg5zXV70a>YODy1lYd=1?;DMhVJ zKfM)3AAK_&OjD^XG2&NV05zEzgS8QyI%qRr8g>A&u-@rm(FLedg-j@&W&Ga@;)$~* z@1uE%X*n7 zuW>3CV=02l{Ed#NL4ax;?FFdl&nX6+1As29*NJ`bKEq#YTxTa<#d6>MV?j(?bhP7s zPfatf06oc2l@&*#*wVWOCPGlan4*>rv8vUT>V$3pBdaF%!RVf{f)Quh`2kv?+ zT<%avlDX%N2cZ6VXreV8Dez;f0}TS!6rBDN9WVmruGNi4pjlMgwQl?5Yn&w#$ayDp z$g*Zi&r~p)CNWvDg|dY=XYk$UW?L>1j>WwQEFRM$ER~hhddTw3yn$-*I~V)y zz5!YY6StsCrLQ#weCwH;|7g_X!a_6H5q@D~&1{M(Ydd3_xno*yXsuEdG)23zKW&VH zqpv~*9l?ohj=j(Zpcz?`C(?@4=UDfQ&XI_>Vd?cx%vG%8=b6g|V@#Prh-V zqDb6y2bF+SAMjvngM10BXxL-Q>yJp#XGJqLt4(&b%<2*gG5J3~*IIdtq0iKpR@PQU zU!htZ<8Mw)@>9(4zn+m90#6kd;%fSY-Ds*yU>^+!V9(2VX|>W;$&?2XXGkWHNODUG zM^cugIiUe_gaN1aRzx$7Qq&xPFRwUa@Eds~?5fT43;a~r@6NsusPDw#=oR!7KZqrL z34f66V*&@oyb!{|Y>8(C4161S#QVM|_|qp&IEcI_{P(SfUl?!cwLfA|iXUCp|Hp)@ zYG-X{?`|g|BfmB&Z})$1ID{lj$!!Rrgv?CkNTWzWk$fa2Dl|!DK@ry#8Yx~(!U`-NGX#;Of-?#9LKrxKOU|0R@Oao+$6VY2>fmklM)*jQ-fGC zPa&oGcVkEHqL{!M*5NN#E8m(uLCFZ08H|W@&z`@zAVoD1|Co zJ$^vh+0SE80ZPL(P%8y#J(3+z$t`4s^b{Mjqpf1d^EVeJH}e-cES75_mS74a%8G1au&ikLgRO{zvGC1g8zmiaY z{rbrl{$*@$OYdoDVQ)|GY3yu7uj*oH^S@OCKdo9GDodze`etPIOrDSc0U$_(&?N>) zklzWUU=86$NVOzl#8t0}og_hkG>!+b2sPSBX>0Ziy*lV7(JGa{P=cFU)>~TE6-^n| zOY1*`epQuP#CO{RQ<~)dOWE6$_sO>RHRregblXon*JvL1&F@-~hgXFlYbOVk>L)|g z?y3VW%*)%pn2V!3+_bOiIOiR}-izzyBOy9}d>XdSMzQYqAL$^sHhBKQIG(ufu7jeh z2D6zbf`2;*pfUEgHI+B>js@uK{}@2oC!X=&9iZJN4#=50m%?~kr@}Vs)baR}MsVwE zXBbaH;olikqkvnL3xnEPq1{jR@iwXB-d#Ru(34)B%#%sIIebXVJq{H-m= ziS0$hVc|PjX*A|RYu3Xhabm}f=v7-Y4#AF4i{}v`-()4s4#zbmJ03n^#p3!3;e>cy zVWgvn4T%Q6bx~{NbRk4*-Taz?$fk5$L+NnnNj7G*#5(4)W^Z9f*df`hx$g4c4eVG| zSf|nMDp?X(j{P&SlvG$`Ni+ET4#~1qw9RXtUJYwCmG$Q=hs({2HM$!{BSJ`P`S(o^ z3;oHN$8NMD=!Vqu#xGX4p+wE78%Cj)&BSw6gx1DcvC5G3E78tdx_%!%^X`-}IADUs zq$+k=<^sHPa@w<}OgVWZhT~TlwN)P0jg(Y>c#g0y22;hSoLDE%mMwIh4)aQ4Tn~>f zEyKxF8W*dN(B}BEqW@g~LO1co^+mS+#xVR{u0) zNQ>l;l#XOFJy<5J$7h<9Mlv_EY4a90=;AcP4BA%IP+GYy+0_}D#T8W|$-1%VnWrJb zCRvKTPeQ1^Rfg*}b?2P30TLQ#ed;Y1v>{{gw0E$YrW|}(aGE73upCQUx&ID<9%%C{ zN4qMxxrrK60yOhDK{C`ttZnabV;p5~&QNIU?@B0HLiX9vd9x&YhSETJX8CmnE5xtL zzEno7)XqGFMgTS(Soh){E#)Sy0M0Sh2O4K?ITBFJumD&2zhXJ2!)pYCsGcY`GQfna z*TCuC4|nD!Ts|t z8D@jcfSXF0Z&AK_7o+*U!PE{cfc-^;T)s_qXI`TM&>FEkux4QFB?gGBW-v|z$__Mv zXY!=+ibFMt)2-98@R)V5FC-->;?~wJpQP_BAtd_1Cp9K=Z7@CX9R)xA zf$?{Bll9HYVSK`c!9rIxb3tD>Of1rqhZeEZrJWmUV&5tE8L?}!%t=gqN6h+ho>RHF zY!r>*DX{g#66zAY3+IZvAuZ90nu(UxxYx{zOdm21Bcr?&|7qB2P9c7(kei0tPC$q> zP~wB9tRdo?_Ltc!$`i9~R(#IST?n~py`!;QMj(fEynsDSl8H;AVHd)_i;Mp?-%+{R zJo5uJPG^>f9Z0}JF%>Z_3^-YzS53Ui&~7(QX5~p`w-aS88DuS1K|w8E{TpXH$Yoo_JS1cWS#nM4HF*x7k)>iG zQPcXS$GmKml~7BRwOk&TfxnVl!7m6mF~a%#ATy#)Idv?<8PWW)e=WTf1YZ)Bt>hTNeOZKPxv6oDAXlq06}J7)l_+qv+DJLBEsGr6g)h1eQj=cR88!v{ebu7Asqo98{g3*U=)CGdS0 z_Bt$~`TzN2$0XtZn-Jg36U>r75XMNYpS7@4}{!B+n21zfJ9(iNSow2pc6dfp9|*A=1d zD(T%Y+8{%mzIP9YHXycW&>h6MIXXO61XS(T6NFi%sunDwHx85|&6*O}^7YYFBfFG#8P$x+}<(nfo+Gok295K589qqhEaPJl;B_b?F#FZ*9DMH?! zIklp!ozK|A(6jEk$l^<0oPuU>x|y!j^HQm*T#}>D+vFf*|MX0g=x2FO@X=BTF=O#-1^g&rS*Q@24;;Eg~$ak`SstI^-%?ZonVm#!xKrr))R>R_?S z)&?7=mHP3?uE-f_5Mx|79B{SRlG5kbAU!GMx$~;W`hL@-DNSna6CY83dmWOg2|Fh7 z6$15+bMis#({^e=_*Xh5CFz|h4dtQ4;{|`ymE=%koKrJmsuTXHNV~VL#aJ2V=dH!o z6=&f_SvFEdAm`Y7s(r9gTgn-55ye@o8l;Jq3Ve|`et*bb(1#%{Z9# zxv&T|VT{0hm}_C_eN#eKlMrjxwD9Wz9MQrgC1Y%d#+oFKFXVvGiZ}C~@*A%n)SN0u zTbb{qydUHqjL{qE_;Kteac&y_0Bu_{i)#V{%cnAy8LsJJ z^rRDKB^6bdLDFKzaw&s{xm~!ZR}t z!^d+Sq1mZ*+F2zD5C2%iXOLc;KMHwg$o>JzZ$)X0dA})oaJurDGXYNad89mbzLA<4 zGwhO={?Ygub^&cHja>AjfNFM(j`<+!gvFN}7`4Po^Yzl;>ifl6alVe5o3uEfbcEoK zT&u?F5qGbQ?ZVc315S5{I$FlAv*i1(=V$t_D_*f0q}|qyoRHnnp*w;~t;l;sv7QfZ zi%AxarNS)Tj~J8*ds3B!YiJziJ-22Tpy%88kK3@N*LBB^1Ziw7-N{!(_0FnycOmq) zlTq>7;3r6BsC76jS7w(`y_FP|aum@@v?qk}j)jVj#R`&RT(#i(weY4_?j7^k1VybV zDTCLzx?}B2V>gS9j*Tw+sm$&zJ9;BD36bArb&dxsIhx!Qt**4EU(D-8YJ%q$9NQvy zlu&z2r+Lqzitol3X-^$owJ@C8+Al}OyKZp?sic;?6I`{=QOMwzWV+2AkEu*g&*T1^ zvHl*DA8-6esqXMYPT~H~DO%m$$=b&L|50egC|mzu{o4!GF4|E%C~;iaOm zGllEF@<4(0M_srr(KoW%;fcS;;L|Bc(qDl;6oQ--WxJP`9Q>YN zll!JXT*XEVNZ|%tu^4z27Gkswj64yBv4#{adKz0)*XWT^jdpgnHc0EH7?b$7ahs)% z9Y!cTBlXmpq1&dG9e6EWxOxvEu_d-&_ruR#88yb)d&g<>@6;96Q5*mAW6eoUz3S^o z`Cj{Z>h>aJI$`zi00WJ~o5$E>+G>rm|Qtn`+gFvT8w!iOj1+3}LFJt<7!p?m8D<#V;+3pqSe{n$F|6lM`y$l+3&Up}b7$W3 z43Qj21cpM0B#*f_yf`({($WN|0DPx^ufjteSi@9JvN3AA{p;N3`ZY?)|?-%E4Qg`lfa~TF@Z#8u}k3V(k-2d z2SGN6TiNxfKEZ&Sk9mMXiF9MM5!^!iEgX(45$2dP<{V&B;+SNUancNmWwG; zWFl8j6M0_M)*e3MDc^Y(ZmF%AO|zl;RBVzOE{kANZKpGRM}2EHi935c|B>ptyjxx1 zx3N~FmXy{p#L&Pc2*y9_PFU0u#nkZ`Cl;`{*=XPXq`J8Rpm z!G<~*4FnM_DUmG60Xc!FiG;SeP}WjNTCsBDkY&rQgZ$b~EAoXpyv1>t{1e(|F=1X& ze8(kpF2TRgUEkL!^o~0t)Xbus$*ujzw=@0nZg-MtKU2lM^Yv>Nf^T^?qJip}9T0xZ z?;jS*63PNYTwn-tBD5N&h^pJi^J$4RS{~l~_;nZc@ve&Ggf(p}BNtQAO0b4ZMOW@Y z7kuIY=zVU=54j+yZ&kj>8*v4|B$~3cI$RF}AUtzA+bj%uWWWEc1#Kf41KMUF* zLmj-9br=qs&rl9DcbYPd;DxP)II``zzEUky#@4F6Hpfi91$O?q#M+Q3uQdSH7KgM3 zxL1N$qh*L3{R-_l7!@Zp;Hw$6xkQ&N60(FyW{fZ7TR>0ivO&UDZ*Wd)1N`=rzIq#Z<_IEVhmv3YzA$ z7&X;|(sS!3zRdjWCdq-DC=l~!J((w@JNY;$3M5);f46w>YfY{@*k!K%M5~V=a}@8A z;?jj*T~HYRn2}>AH0JB@5pi|zkM<)2Q(QX()m=pp3$9M?I~}O&B-gD$G-Jg*YeF~3 zq724cfdu+4xVC#UV4@C+FJ{%a4OL`Zuu-!Kbq?r8AS8zK&oJ>`ViK+AD(PX`u=o;U zVvw1mCxP5`D+rc?bs5kO`JUk@Jfuf9Jj6#jJmREE4u0l7796h9U0>XlyMS0b5sc?b z+{nMS5BZkLOTKX)9OJB(BxL(DZmy7Fq!gAXS%k$+YAf`JEokMBc79qQLtrRXm~`XRhjCpm-C(77G*>iaLnRgAutth|byUl(KERUTSEt#N{F9jSp&k<- z^Ha4e2pcn#x*>;*SfB^6N1V#m)&M0PJla9_a%AVX=qdI>KF-nuXagF#w_AHf{Wlj@4)-#yG9JWyAUi zmen`*e(f{846iacQFvj(!-@`Z55b2k?TBL)FW4cFiSz2JDEN1tnfiVR_8i|y{VuFhm#FaUGyxWBesnt!gWc-V> z@HEi{jclu+m=OdXQ>UO}rGz7lK7J^r@Qzov#?BC@XHQ6BpdkLGZli+rZ-Me@kybi_ z2bizQjOAov4#vMHhloznh&lOmTZ<29&LcQEN(>7V4q**s#+)K@)XJ6J|CSsuuqvnQ zPw?wkgz&Fl)clrx4G2(_k%Yjpg-Q`=QG!UZL$+}w2=GaM1LcL?7a=I|2`#bTTy1Zc zbCTW5cW~ECdOzgcAIE%rw7~O2{UV=M&r$hk()jjQd98?~kD5_O`G^6pzw;CP zP#fBbD61*nw|;F8)AplJ-%(%`A7URm!N)OwdVDPX2`6o$>aN=7rm9wU>{@;7pnS#a zuGt62`ylo!-519z=2v4>ACe#3{8ru5M*TkQ<3G>`nbIae(XuG&Y(`#|bZy0b>@QM}cLw;9C46UasI+C}jZK^V*+3ww26Ps+k8#B8!yI2_u^`ftSrIqd}Hcw&Ws~W=0vm@AYd2 z9VhD%;>2rBb;7h8SEkU<1FHKLg4Y>OFG|aIP6nqd0eHY=(S%OP6>+kt^BhCU@)D)W zAyZGH>x&w_0o`ew=7~103QJp*Nex|-=HMXkgKtUevGegXMoaOiNSQ*>s#}IgQS;u? zB~>&^6)jPzma0&`sziu4c~m0CHSH?``faQ=J56V8SnIZwt9LO~Bwl(OU|{%HAG3?9gaXaxT4Z^IjONes zZ9Ak4Q}S3wM)F<5gb?97i3D^-6#H64dEBT_l*^WYYZP^cxK*u`@92`;$~;;kUnm+g zK+!I`q`g(F)Gca+c7$)2QRyovDyXVytTmLCyNWA|tFy7G<=`)&P=bLIjsw>{XizNm z{xB4hU|Gh16Wu)pIAp{!6n{^K&etCeT~M7P=-deyr-H}))FG-Vt}7^YRhLgU_qGP4 z0~&a>g}pddQEa0I5o>wo5FjsSwGl(HJFKI^`ax0imN{-{i`W)%AO|;$Iu955wxHjk z8&XST*qVEhDoeH36twiT)s?$#bag#llWcRozIiCxE*kZM05ZeytJ#_CP3qrLdWBx1 zvW}($_xDg8N>1`78i^MakeV8CEF&$f7l^!Q;o`hS@dLoxp{Sya+HL!j8l=>{}rTorPsRb=P`n z$YIx04mn-iX;4;dWyDj^R#4ghsxKu~?XEqsxgmp})!8y&go6tuv~EwzY#{GPIxeyL z!u0)IFmWT*)!JjIKHx+iuXABIud_T^8*Gok!#q(~%vREIi2w;^IVqD@6hc!|SF5{o zTVd!=w3D2HO5(|DdVSNyxYY&NL#^e=RSV}zaeexEDCtXy#-9_| zRXb>)h_7>2)n>~BFsthp={YrJHFf131w}ngJ@c!2`3~5N?$7>zR?Fz>49h{Tq~;}d zD;wHRS+-t<5e2We7Eutwc@~8c!m1V1dDQN=v#MBDwvlI!TX>y3lQZcRnB-D{IOuVp zL{$M~ef%hU#U3?7Cve?LjtkEcF76ueY!`hZ1;RFRA#z10fZ{ENL5-gLlFx6n9c1MUs%#xNCqIDFA8u&q> zKsM~GJ6l3M2eBPccX8v5fjNQNrd$%N1@=^w8n$-LIC}J3bqO6c68HjB&E^I#S(#EE zym@VO6aN%pC{KAyupEWP+|G6zdB8Ri0;^S3EfRXjGy*2AzWk{S8?ZHU^k3yaQ{p=Y{qtXiv;EBZjK=;_jYExjhHu$uU6j?}#BX+n{WdNe`FCo(bjMghqQZ@& zM%Lkw!uBy4ObsO5K;D;b+(eQZ#!9lpvEW9IQHdG5GKrrRb+jmO)yo?d9OA~lCRO8j z6k$IxnZ&j*c-8wjjwuNeTrHvfHgng>snc6DwTG$`%>C_Y*3+ZsGqf9EC*uMaY|9`u zyC~*dVddRujg`FBDI>H6j-yFpaHB#=Z^^*(-rNG-(*2N9?=v_AGd$O1X!N@ih&wn4 zY}u(UJHEfw{thbTk2VYDmDxbC}qxUzM<qY85pE{I8?6d@@FL*H#a_ZZ_iEaIo!b6}7^(VXTlN zEcE4?>Vv|pZ0?q{ML*ymeP)zSZ9=B^yh}#~3S7qk;i9WWBSP96*7+Sphd}~c6MCYr z4vwqgcV;@K!`jqw4n8!H;9A-t>kBpu9^)#UQlqnlVFQ~rm=(+EiG>p6><%iA60@&b zd#OPAQ4{)FI2rB9ZF z;?H8~{^7=~O$!Nux>`9EY7YkwC+6kvCa<0w3e4s#{TuNX7EV6Zj1!zvB)zJ)PLx+Q zhk>oPmrJvZDGauM3w&J5zF5ynqLRr=5hgnvucz!k8fq;q;{=w~$;4Q?@+iF~2PToj z5&mrW_tyrS$s>VdrhDHSW5;}StNiIAx$dYr2`Zm;!zRvhln>TtINidjQ@haRiBxw) zbyPSD_q=yHujGU5F-f4m@_Mjrl(1$Hf8WR@8#6TPE45ELf>gSrBMq7b`Vax^7g&zt zA;4#-UY*jVNo-}d32S=bAC?cKKI4H9tTRc}c3EN?9@pBR)yDbvBFTFFf{W8XLg1|J z*>(aNT6lW^RumKiwosz^4WGFxK5f3Fi-scQZUK_hhArfK)VDe@b|_lZwRB(Zb*+PL zv8@#hD>LQ$p%zdgx9n~l?ggJ1KO!3xbE{Np5a(fI+suaT!nrAzD6k!&xL>3Qc1x@z zr*pGI@a;*}qN&Q1YU^mLMej)6Cgq|yr`;ZlD~+=D>(|H;DKq88huV_)ogj_~Tf=#Z z@K&ej)N;lgwkqAcu*LrxP}^k-HqJ_L1TG7rH#MjybJ)GFCjsUWl4r1ESY31e60gdU5Az&nBDWWz&EoFP)>ZT?>+6*Rvf?$gT9J-T(B5U;72Of zG4-HU;)Z^CttMLNrgb1A9PPk{%e$57t2e8ZiPgq-F>UGwU;bcM!pFYD+6H%{TWpltT z;ah~5UNO>jV(|N_SiB-(-mX}Vn1)I{A1cGv+DJuruAb)K0jxL)`P^Qxg>vUeyZCdb z6ugy+?4OzRa9F%TaaNBQxbvrubxGu>NU-enCat(F4#yR9fJc;*ZY*BexO<`c;=YVq zO^u^B5)D4f2~oa~?736Ted#C07(YRAjg^s^3q-W!A)`tWfN>$G`Z;^D1|@>!%asA| z(K`_m3t}YjU2As%>9*G40dN&h9cs>ImOPr#AmS5`Zr?cKko+@BLFNR$m~Ire2&%$B z#T^!`J#yT}(1F?AD)x+Nr-6uCkh5)!c;8kZ~N~4r_$8 zi>UW2#^6hvC*!4|yRz7aGh_bdR5eDgV1@OKw<~WislQ(AaY8u+ZfEfn{xLc9{Z0rL z+aXHKKFYRRSpE)%&@Nl?atg4d7qeZYj{HojS#<)PusZGi*j+ox&hjRJ!wRsCgbI5) z7HH*I-Xx<^c!nWP(|^mqw>mODL54a2(9Wa5W&De1VDSI4`A2n_@;!*G81p58PXu$J z+#r_fCbUG|946>Q&@B%Cm-M0ON<>;62kKpa5cA%T9iX>uqa*N7j(yjb*-+NWVyjuh zrA4v&w4+QgDhNik0qw1$PH_|=x&;&z7(<^QeU8OU?QbY?3b@c>`z-XNWRBwy7eDJ_ULi(Cp z<}ET<##F|L1Yt+>R46(cRsL{(2D^+29ldm{!)K36vQpZp4PzC&$>@x;I!t$KYhUT+d7dsk&?OK}uMp8mFFG z+bV2EYCi>;1P5O30IQ|x9I92(@-i=JDo~9Hpp4oyu%y4J0EF;1Q)^PQK#M`vl$;*fqz~07SYw`m;tZ*18dRWiVa`FL6#%gmstePngYKSH$|~#Hed*9vY+jU)vsF`4 zmr+yOmR14Af-5SEi`zq2SO+R-RGx$-zyxWFqxvO?&QIWiuX;O~q2j@vaBcnziuq?Gxt ziOGhd%>UcBlXh#a{q9oyZB$@>`%rUR_VI=7GkoQrvh%&`_}v)e%=Af*eP#6aZPVEQ zc*3S-`nJQiKJbXn#+8(#->>fEufMS_o%S^>mU8!?eTsO1U~@&CCWfsGM_>i#lsT~6 zoo;Df<47`&4(?WNS}9lfmu^ zJ!53!8h+Hk-Wq@8z{WNA>Wr~7_6m&Q1Dr*$d7&-vVf&&j0Bj!DDsRT-{RC7L4!zSQ z<=Je!1=rie=J@VH!KaO()%Wa^#3J;a`HVe*<%zzi@TY6;plc4OQHaAWCRn zzIb((edr!eqGqiNVl2Z7pO3ir%|^nt60C5`u)*Dg?mfxb;r3P0w&~T_;LgDkU+^Bx z+yp9a>f~I=0;WQ?;`MUQE(7N_wUiFeLnq(tFqC&djk^xG41J0L_*o2u3rnEL_zf>_ zt;8w*U4UprU50;kneUp73z}ISgupmTuci!{pc_z>t+4aAE~(*$qy4xJOkxIzVuq;A z0<_n`zlL{RqJ?o@dhKk2+%`d1_VrHtnMp0QF^AK3;|+fshCI2ZKcTjCuNwlP?ExWt zm)ud%4T^qT8gJK zPwxJy5|7~JQFEctFmv5=%04t%PP2ihb|}h|T_MqnBu*=!7~vk`PC|Vlka;-qDDgJO zfZ~PG{IVN0WBCjt9I3O&{3aZ22Q7Y}CJ(3=NPUVQ`Gp_i69{{P==RuoJi`eD8kQHo zHuX3yRm&Y=p6!C5^sx;{8@-!GL))vKyqmt8X0`pfg*+P?r*r26GV@8RQ>UnrT{^V~ z2wG~4T>n(l%?peN5w`O3wj9r77{$)ADHO8t1l_eb!;C zU3cPv?#-3~5+%&V@EeRML@htDsWJLu zCMWKOJFCV>_qW-wVLB2sCKQ;I+6`JVd55mYyuofwC|iUv3MpN^ix&n>3U{Cw!VK-f zY4-4h*A>4^;uCb0uO@ux5#>6~SC_}`veQ_0ePyzbc)10$iH%AT#k!9&!ES*hhM)~$ z^k5clktllc1webOZV(N!ka~Kg51F>ebx`vZsGVa){l6q+62DC+c{ALnZKLmdhta zZs5g&+fTszR=6F8znJmU_Cc**D)i?$wK4sQdgBtj7$%T74;umDgtZQrRhR&?$$-dW@(?qT( zQuHg9*>1{i&OETTfTBYX41h_@iEX&x{t*Q_9{$4T?31U{7{56xlmMkU9`J;c zNq|Bac`OET`DlWaH{=u(;IuPK#bJO$J1FCshUBB@Uuc~oH5>;CTLwPcHS@`YbP;Lh zPIF!VqaEE4)L@*#(=B%v`q}vpntCX&x92L$VeY`ekzB_K#ZTatjpd1w{o54loA2X2S~9H@_B*XBcMsSM=Zr#?qt_ z+5}5vcK|4qoiR#xC=00YFl-PqC=#kKGyS*$4alw_HE_ zPFUfE*x})aVnX{^Fi7fl#mX2Xs@Pm!c#VBbY3$G}5r%0%3zk@ib1rOQHX8DrvjT0i z5}ruuNiaJ^$XMu=Cn_Va>303QAoM>13UXdJ!XQwcSs*)RF%So~coAu!LdM+>uui1H z)0KF0!6*A^x)oHVi!W0@byI#Tfb|Uj)=CCaT;bLbGL~p1bXV`jV zxac0xVbTrJgxtRd@fI>PfCJQ*(yiTg%1BYfBymJ{u z@+^>sa1=%pDN+?6rc0B;hD{Mczdf2uI4=?)i0I0ci=xd*$uiFr$B^vfmf9>yb4Ism zI~wJ0*tr_9hcV%8rKI=$`oO&h)(mF&n;)%DXp9qN0Y-O6<8WB!<%)r2VGEL<^Vc=p zs&jUrC6W26JM~`&qC}1sJ0pjA6k_m$MU&tIV?5{S=FfuNV?~Dx#8zK#Wc+=rbgwVO zS4;7k7*Q+m!}a5xgw&fZKo5w2lsdREg&X7KW=Ox?h0ErE#i;|G&M(|zQaW?--&#W$ zHK8u$e;`$00T#5zI>69n$&A|&6=irX>oDf?We#+<0nn!`t?^hKq3nIYg#*Ga?MV_j zV~pPL#FhOaok2u4=*e&+G+8(UU9Y@l0AmlZY)aRA2D?Nq$uQwod95dqR_XOB=zC6I zKIj=U{45Y$54bpG8f`W)){MOxjK6qzh2sl*?@-y)`}i5n8kOL~FEw3#5o72JmWX0>iTK#(0(&PXVx`) z?rbvtk1~EWGY>z#y`=ygXN>nwo*Ul8%yX2(V4j;9lC)!_!&;u3u0)rP6OM&U_qz{K zgAa%iyqTVbSN1r6^Y^uZ>x*AG9y>d~G50?Oz+upDgo%)kUV^5*Y3Rq%*+H#>CM?)< zn!IilGb1e4WIExCL@<23woO3~mBSdbd*)h-3 zA?P#BHt_5TAO^}+7P=x z1X~b~&i^G;?K!31&iU<~IDhuRLhUV-PfQ37=?2?^UY}`2G}ebmW;6|F+hm zZJ;g(@EKX;Ansi6nSLiWp1I(e-gfEylxCR`DS7uj(MFpBjsU#vuhIl^B+(GQpaVXn zdZuEh+^J!ukd-ER?7Y9R;d@aMHr)G_vu+zL8uB$9_Es;v`7ZxNbZMc-%EI+DofOnb za$^yb6mD>C9FXK(@1h*rrHa2eKsnc#GAplghtNZ}si$j92JMMu{1pZbGv8lfP~+HG z@`yRJNjj$NoY{e#p}CJk-fMz7a_rbAn8accMpO+k=S1Nnm%~H!iL*WZGdE1gtK$-p zrh6MjRN~o*-X@Ye%?!rh(}!_(cLI4IQs|1}r~>4LpyQIN+(L&TCeF#_))2x2B>G4} zQ!tVnnEmrmmV6aC0TjI89j6%LqbF6)-el3c8T}(n;g=1UMmcAabQUS42;%h^dgRLK zh6qEz9`RTCh{KW-ehbs)CVmx=|-oRQ|eZBQ%1IoZc z+PAv7M~)S6y$JraN&X>Qv5}qEcmuDs_b9QR|{6^k9W1b$kE>37E zO59g7cFg6!Fw#%u5sLW-l=|a{# zko}9KY~MYL>!>bP58b+ks@C?GZMI{f`u@>|5exu@`qR#T+w%Hgc||`=U&fzov6U#3 ztWtbghn;yMdy*3^+caf>8~3a*g>%Z$IDLv+JCaupC7mHr@tRri#K@NXP?gy`F8%iG zB&-&PZacML|C|m9FfMkLclJUp;ZT#gcoeQBLYiXpa=Ak_@tPJFBg0X&X?1OyTo7wr z>LP9qgwTx*S$D?ylQ4&*%1~2iy)`HkSU)USZfR|+I=Rzw?`~j`DmRb}`s~vZ7=rux z3`o2BEj>dPSo{s|BQl1qOM z&g4>M*@g_LbjATFOk){hr*z*bvwCisT|PY1`Wv?Xp{wL@{5yw$8=?b@S6qDH0+w3usD^fi#@Y^lXH>!TxM^i^H-B~mP z|GI88M5Nd|C)~dE6wjmcR<_>BoB=btfJL2d?@V9s1Ppl|l+ z!7~R>l1f3^R`FdswN-p`k=XWm>(3kT%8zzLm8KGdy)ED?kx4U5;qD4{XMTmkU=Oo} zzl=W~@pX#m^&-U1i*>GM-GBe5YRuwR`vD8_*RQu96Zij7$Dm~D?DC&Fh9ZrhLewKy ze|hgo3)bYyBwA@UwhLfy+Ob3q8lOF3J@%jCy>w55JoZdaCVB$>< zgnb8)$+Ta6iMXIbSP~hJ5*7fmJX@iKL^jLxMoSA}$kuOm<__)-EyacF8}rci37a$P`NUh*ntwctb2vTwE@%1+ggC`kqc{ws+x(4DqYH4fm29y z`Z%*n^`n6n>8(ssh3fNoqu7xq)l9Nw)wr`tHNdhfemZH5Cuv5(vwIv`g>PYiPfw#S z%A>B5KH;fBMUU!$De04mPQ6WVSS-&XaScjh%30M*_Q)(rn`d*}JnHhK)TcW7q^wiv zh%ISG-7{&tLRGKeUMpn9(*jpN_tp!(>I4?b{&7}pyU?LyJcD;}`gany%69&J2$j7m z$7vpcy|PC~SaVbY7vnu#QisVU8fLo6Ti!UY)A5MPTlaVu)u%w}Gv($@o*-Az?Vm8Z z<1DC~``IF&gif%b_72m9q5OZOfgLJ)aTT2-!iL>4kC0`Z>PLi%oHX>hdIR@ZcefPc zDBYXFj@#rg`A|10NvUB6nF{})LICxt_p?cvp%@%M)R>;Slr_Lm_$nUJmUg)X4Fx^M z^78iX^6vKjwH*_wZjaCKOrG5I@Vb~*5urxJw7zF1By9D`mcz#$G)Ztn^W4I7(bJ6< zUU1$(xY}&6F3-%g8%UR@FqdbSr|q@=7uf=}H9hRBdikZx{=ZlylqWk)O#4VS84 zAj+!tKXR$;uWi{3ZWyX!PSMm(mWG(5t-fs9K3QFFHh1@@C;M|CKR-#HXWMn7%M9zv z7dM;hYghx%U)G++s^A@O4AcmDeo%DugE-c<IHu)MT3lU7S$g3(e<0zT3DArZM6bqHBB6SqSh(mxy;k^J8 z7jcS!TxMhvK&*)N9|5<&FQQ^bc`jrTlNQuEy5y5`KsrSZo7Hi)crhb= z!DV^Q6x}_hG$5^}HWW=QrXq1?V+vf^HfQ6PI~C;gll=&O#h&%P-!c!oKz}`}agX%S zBShyf9FhN}+?Y7o#fG^uI+7G;lstw#i@BCb-Q_8`m{VXPt_pE26eOUW%hVNu?~nP; zc~E3poh>P!>7s!!W;tQx|F6=%1D@*d|DS}cFS1o+_CYoo8QFV}WM5qSy10b0Ls=Q6 zWRGiwmQ{&JNQ6rvd#}m}B^kft(?_~i_5UA_`?ww+_dK7k*Xz8_>zwyF?(6jmj?mWN zOYxs16)(|eHa93MU3>&Va`2hcTtf~%u4q*(C_NiXP4{S;iTd!#!aUodz|LYV(zar4 zhPHMWXGIMp(>b1ZL_-n9{mCSjSaJMkb<9P@vpr5Q<32Ao9en0_%cGf7jXvf|p4=hE zmp5J<5t26(6(}aIwqI5r%J-*55inZi_}Q}_2I+sY@=%l2_ZY-+D33DNG_;Z!3(;!i zEEZ#A`v+b-W#J_&*Oo#|adq%$@1p27~ga$$^CEYD!w$d!&!-RyzAoCH|B5R4eyea(~H*dsVKkq`}TI4 zGL(f+Q!hfK(Kzy@(cu}nxets5r9%8-1*g=g84@$?h2|~^D@;W&7R4}oT2JH#_JNw` zp`wHx;tX0n$q%QK9bPRe!yq)&DwF!2CReT#yrJx7%l?#;-Z|9?6+BK%TW9n6d$>5Y zZQ1N}|K%qVH%-gEg>)`Zze~?PEFzXj7k3vEH+$*D#m4R^RlNG4Oat~~33S4RDw6SZ zlb?e6Rxd#DFAI2xG2q0+!Us6BuMR1PaxFC{zty>8YZ&zAn61MyQ9sN+hb#~`IiJvHHDq6v%Tn8gnVIo|+-IIJwui9Z{<068@}a=WQJ?ou z>EC#tVN@8YSu_fk%$qOqPJYqp;Egqqi=vvE?=7^;$vzc7lD0y+W{}y^uJwcffh27_t~dTteqE6i+Qp|9C-P8B!cg|k z!zMkby5r3#lvbo^7Tk7~ZH2YK(ZnuTPrBhHW6k`i?}reM0+Hs9@pt(Yl}aOt-`W+$ zBjH0vf-9CQWRWQuIf%|wpWE^+G#uGs!5l5r9NEg?t*xDQjb$yat_+Xpe@-yDR|!5$ zGvtx*i56RL+<~y-He%Lc-n^Cc+&YiXCauzs<)mBdPXvpxy}I{GF#4Jb+e4I44|l|) zcO2=ZKJT7M6?cs5akNN&e}))+Q*6;~(H}FAz|n9`r7|(ky;}_{ySmU&_rWY(_E`H& z5A3eNgCSE^GgOFU>?l5G`-*xEs{{vz_m?Aca@v&!VxLr?M_n^NKr^iYM+#_u{j$iSwo^fe7JLq*O_v0T2T{vtw6qm_F%-nBS*-=vrQ zG{kMY`k;w!NI}q7m0edYv@t3#hv`mGI&s7>d8!qiz+#qyd+MF4%9LC><*CZGbRSXg z8*djJB%TQR;29?lpCxD2&2cNz7~y{XBr5#1(*hGN*}^f+Ov%=>v=topzm6`Qy@cBr zS*J0_Yjv)_&DtYl{)QsLH!PLQK%g%T^;{*uM}EZSn~2rjAguNmjND&CEQXeSzkf=b zzrEaOkv}2m8({W!nPWA!uKt@`c9D91{szy}H}lApoP4(F(2z*SR7;06WFggta{5c$ z)~T{8TO#$clA2`A)@y0ZG-c*rd7kEL!-2d zg`0J+g{#`jt99Ln=jQj=MlzgXni0!_7v?vEl|R3@9iwAtFyg=+4>MJZiWntAhP;4$ z@XI#zOFK+o%sKrJU;C=HuiX%g{cXpW~L_iu#!MB}xBK*Gh+kctxo!;_MiL(5zr8!h8I- z!pOszg5LG99_M1eL{j4S`h^pjujiiAPdA_Oph~h1)M$M5vC@{oM6HtqrF}Q2W>LxF zDWxWjAXa5FP8mjJKh6Ngviz}CwgeTS_k+q+LOp}*RjH-bHis%}alT^=fO7+k0qZLj z{_wJDL4QfhE4MK#Gp`LKpYfy`ED!6LEv;rByzDsVSoO7g(6H+3`@u2)S?*zvmpGR( z{5o(tF@39WI1m?Fq^YU=R|G-edGFFV^7-&F{yS=|c*`>`A0_en zT~{D0>}s)C&%U&F@*o0wE?uJG$6g(8=ZKGM?|j%7n*1q#M5YZtF+Tudu_&07Mv+X^ zZw(L6^K3Qq8vVzMcCEj|MYLSY_C!c_$)JWJ5!H+5wAEs`>CZ6Ix!YFGF~MWB7-4j4 zq&{S)^fG&&;jq1)bRTRC9mv7P_h}3$c=Gr<-Ge%lj_kMP^;1bU zva>ZWX4X6XPyzEvxBcAe)u_L&uY95p*Xs5wO8-jaQk`vd-IvPpQU51#kEG_y{-wg* zH!X`v=MG^xA@FYFiZQ+%)|47=&yYeVc6%sv zgeKqP(s{4*M)Mhm3`a~Hq*9a&g$#0<%k!Fx;DKBQBck%>)!J61>cwn41J4{eo{WsV zvf#9+(aMU+os9e9@VW({}!(UgeQ~iu-1VqACv+c^{+Gug(goymAM3F?>>kxBY<}qQqlb7);s`Zxb!V~h@ z8;$yFOX8v&Ml^8w!oowi#EgeIfd#Oph&YFhf9m22mJk1s=f$k-qPtfpv0U&v4qw zSx8u_)Qn%+|8z@Wiz**iMBLjY({a{Sxb&?9Y7)CXytpSX+dELL)7Uzd4ck6f)^|0xT*_h>$m7|Mis4s8YGAKC zIdo@|-09nKN=v%?(fkULUnMsJyn|DCx#!Ky*Hvt-JK7?tv)1E8QaGRbZPe(WXna(1 zyvPjooc05B(ZR}yW_;t;SY4cdEslcoS`x21lL-{oOrbB{%LVZ&v|MGb>-aunw^DjD z3v2dzLw5*8zgFfYPM2blC{!Rt|253!i}(g5($i1go;jb0C|$5CDoD186S6v)9AJZ& zk{wd#@^qE@UBhEys8-!1uQs%I*89wD8kHgKjDeBi?{MQ&v(==tnPhAJP9fH1{O!pf z)4q2>`Kx1+@UD-lI=|!#5H1LLJ&>dCat;cu^_J7>4}RA;`W(Np4NgnMhFV%kA|9ER z#`U*p5Uw@NS{7y@`Fv_BjL#U+Ffs8&HSB`tY8G2BYwd9{A{I7ea8*(9s0Ae0JknSU z>y0!E=LkFgoGQMnAkhe;FY2)v(L=!lrAiz~0&JnBu_i4x7m}Y<$a}yK!TkAaJxuj_0}1@N8f2`!a@$ z!7>hCKC;5CWA7agVn}!tBFrO&TW{w`^-_9X3DfTtYutHddTxJR3Bx9WykM&L%#BL| z@lciprXYM4vIw^Kmqi*5$qV=&s(Ba@#ouRI7ppZ9*3MYGWQ%jLxE5AS+|KOpYUunj`hyehg&(F3Aq9&&+#i^t&&kciI zggS9=9q8=-Fx~DCgQP-S|0WgU`ZuZ2{(YH#+t(8A3jIlb1j8LjdQoKHMiThW=}C|k zkp0g9oPj`|FvnVbwWyvmB;vQ9Td*%iV{kNFj6B(APa)il!{KI0{Ucc zYnKLRY>3M!t#>2UQ7Y<`SEKmh{e)Mqo5I46cNiZ_AC2oIzC`ETpv!J`ajdUpV1>Kl zJa4G)7oU@sdL4#V90V<02#9JN7M z64n5psUgWxFLEeXuGR4~FYYCrOb*AO6UDWMuvxHWs2FobX`SO_;BaVclUjQbp7Y8*3>Ua!HD3Jzb^4A;G;zbK^(D!x#F^#gD616X z7V}HEBU7Vld>Ywg#udS+QOtEkRydb`rT}#iT;7Nt^;GI^CWU61 z$Hz3WC%C*KFTMU!RGHjjG-H+Y8Kg)g!z!0#p+CiS8;%J{dF}yuQe8uRcztqI`3BwO6V+`f@zh8E4U&DdzL3g%=sawS5p6DG;%9bv3iO9>p_UKE=7Rm5GYW zgSpB@Yr?_?7jbS=S0V{Fl;1u?5njC;J+yv&Wnh+0eXt$!#6yD7>Z{IcPX8Ig zd6)zH;R9E0p1SuY1pIvR3}sYbO77DNKhz$lxE)B=mz0*k3~3=vR`_Z!-52EFw<-@e zArkuVM9IF;rQcxru5tb2QmuA7mL9>pi7se*Zk91DCG8nm536hx>0 z8VSZHsm))1hScqDM=+v6MC&PgRqO;l5CmT2o1grzApf7xMYX$Zp3QdXLo_eYAx zomjP;p+40h>EHiAp}=i>VPGHwhQK}Fx(W0Nuwl9f znmoi3xP#Z;lGO#f7{mbwwP)Umb;@E7tn;oAXRvGD8*A;;BVSD5s^5UA&Z23#?6M1M zJBCSFZl+Upgj(4^tz_WtPQZT^wgTMVL>~92Q&NCASAh|P(1^4z9ze8}c|!?M*!-5{ zJI?7(R{#MiurfCqiJa$wB!8m8!{L>y!$5=gz##t04{5XuylpGL4VqZSrcNn z&uH*XthE|}(P#uSiwBKD+2?O4_8x*xOMPQ?fKCa}`O(l90uDfz18aj5h$(gu$Dcav zO(4Oi{q_tXpa2A?(FkHg4j_=XbKJJh-pHgTE>7*h_Km>y|DYis4L<<+H}YDhRUUZ< z9RC#IkA2w|FfxE6@&HstJA@nD#uWnl4SWg36fY8(V+@$%%oa9yM+RuzJP01{hH!?v z{h7y)pBm?_1v{SSAkN94aVXq5fJ4a%_#|v|P@QZvLG2GWh=n8cM-`CYC^c92b;=)% z1U(oDE;K?I@&H10cbLU*r!Pg-U3@7pb)SOW=4L*gt%eK`$vyyE#}bSMkl)JC&e_@y z;`AF094uO7=z#P`z{vA$QGj=3Kx_U%EKnEsKbi%-kIC#O0DdS~{RPnQCGH=D?*R3H zTB$*0AB(-Bt|qPm2=o!@`gSn3h7F?T+8!+wb_+~|7zhQ`WVpP~s2 zy9bya=g?@D$o_^#8{)wXUa}B3h$b9t!S+@vi0Bdz3ve(^;LyrwZ1j}>KWs8^m^0*n zLkE>R?C;M3hxi7L@X#L?3aI~*cW=cCAE;NBfYBlYD<=BNEY9>d1e-HQ2kHv$g!bmK znr1cO2E$bhUg(RRGRMI@KTyH0X`i9tYbL+=5DX0?2+ZgYgf*uRM%ML!D>W%;2C;Jjj|uW% z>j-h%cMZ;O|FWC{d{i9x3^J5WS6a%VHM}`oFyIgxZ;BcN;1Zx-Zmk4%c?(aoxs@ve|{XeMr4^wzN-H z*PW8@%wM?8vgUXo%g$_uJArq8pV=gU~FvIf&M>la<=Z&_rE;?o0eZX-`d?7L^AX{_WiM#wxGZj$;L+8Z?XQnbKe5o z`__F66B$63wo9`e!u<>Q-yQt6#rD4G+s6Hx`Nz21_kNlx*dXwO#UK2{f~Z=61dABP F{{hk}j&cA1 diff --git a/example/simple-kv-demo/README.md b/example/simple-kv-demo/README.md new file mode 100644 index 00000000..b73ca5ba --- /dev/null +++ b/example/simple-kv-demo/README.md @@ -0,0 +1 @@ +obkv-table-client-java demo project. \ No newline at end of file diff --git a/example/simple-kv-demo/pom.xml b/example/simple-kv-demo/pom.xml new file mode 100644 index 00000000..8d05ae90 --- /dev/null +++ b/example/simple-kv-demo/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + com.oceanbase.example + simple-kv-demo + 1.0-SNAPSHOT + + + + 8 + 8 + + + + + com.oceanbase + obkv-table-client + 1.2.10 + + + + + \ No newline at end of file diff --git a/example/simple-kv-demo/src/main/java/com/oceanbase/example/KVClient.java b/example/simple-kv-demo/src/main/java/com/oceanbase/example/KVClient.java new file mode 100644 index 00000000..2878548d --- /dev/null +++ b/example/simple-kv-demo/src/main/java/com/oceanbase/example/KVClient.java @@ -0,0 +1,228 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.oceanbase.example; + + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.table.api.TableBatchOps; + +import java.util.Map; +import java.util.List; + + +/* +* 此demo只用作指导使用tableapi，测试前请先使用下列schema在SQL中建立table +* */ + +/* 测试表schema: +CREATE TABLE IF NOT EXISTS `kv_table` ( + `key` varchar(20) NOT NULL, + `val` varchar(20) DEFAULT NULL, + PRIMARY KEY (`key`) +); +*/ + +public class KVClient { + private ObTableClient obTableClient = null; + private String tableName = "kv_table"; + + public boolean initial() { + try { + obTableClient = new ObTableClient(); + obTableClient.setFullUserName("your user name"); // e.g. root@sys#ocp + obTableClient.setParamURL("your configurl + database=xxx"); // e.g. http://ip:port/services?Action=ObRootServiceInfo&ObRegion=ocp&database=test + obTableClient.setPassword("your user passwd"); + obTableClient.setSysUserName("your sys user"); // e.g. proxyro@sys + obTableClient.setSysPassword("your sys user passwd"); + obTableClient.init(); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("fail to init KV client, " + e); + obTableClient = null; + return false; + } + System.out.println("initial kv client success"); + return true; + } + + public void close() { + if (obTableClient == null) { + return; + } + try { + obTableClient.close(); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("fail to close kv client, " + e); + } + } + + public boolean put(Object key, String value) { + if (obTableClient == null) { + System.out.println("kv client is null"); + return false; + } + try { + long rows = obTableClient.insert(tableName, key, new String[]{"val"}, new Object[]{value}); + if (rows != 1) { + System.out.println("fail to put kv data"); + return false; + } + } catch (Exception e) { + e.printStackTrace(); + System.out.println("fail to put kv data, " + e); + return false; + } + System.out.println("put kv data success"); + return true; + } + + // return all column of row + public String get(Object key) { + if (obTableClient == null) { + System.out.println("kv client is null"); + return null; + } + try { + Map result = obTableClient.get(tableName, key, new String[]{"val"}); + return (String)result.get("val"); + } catch (Exception e) { + System.out.println("fail to get kv data"); + return null; + } + } + + public boolean update(Object key, String val) { + if (obTableClient == null) { + System.out.println("kv client is null"); + return false; + } + try { + long rows = obTableClient.update(tableName, key, new String[]{"val"}, new Object[]{val}); + if (rows != 1) { + System.out.println("fail to put kv data"); + return false; + } + } catch (Exception e) { + System.out.println("fail to get kv data"); + return false; + } + System.out.println("update kv data success"); + return true; + } + // del + public boolean del(Object key) { + if (obTableClient == null) { + System.out.println("kv client is null"); + return false; + } + try { + long rows = obTableClient.delete(tableName, key); + if (rows != 1) { + System.out.println("del kv data success, row = 0"); + return true; + } + } catch (Exception e) { + System.out.println("fail to del kv data"); + return false; + } + System.out.println("del kv data success, row = 1"); + return true; + } + + // insert or update + public boolean insertOrUpdate(Object key, String val) { + if (obTableClient == null) { + System.out.println("kv client is null"); + return false; + } + try { + long rows = obTableClient.insertOrUpdate(tableName, key, new String[]{"val"}, new Object[]{val}); + if (rows != 1) { + System.out.println("insertOrUpdate kv data success, row = 0"); + return true; + } + } catch (Exception e) { + System.out.println("fail to insertOrUpdate kv data" + e); + return false; + } + System.out.println("insertOrUpdate kv data success, row = 1"); + return true; + } + + // replace + public boolean replace(Object key, String replace_val) { + if (obTableClient == null) { + System.out.println("kv client is null"); + return false; + } + try { + long rows = obTableClient.replace(tableName, key, new String[]{"val"}, new Object[]{replace_val}); + if (rows != 1) { + System.out.println("replace kv data success, row = 0"); + return true; + } + } catch (Exception e) { + System.out.println("fail to replace kv data" + e); + return false; + } + System.out.println("replace kv data success, row = 1"); + return true; + } + + // append + public String append(Object key, String append_str) { + if (obTableClient == null) { + System.out.println("kv client is null"); + return null; + } + try { + Map result = obTableClient.append(tableName, key, new String[]{"val"}, new Object[]{append_str}, (boolean)true); + return (String)result.get("val"); + } catch (Exception e) { + System.out.println("fail to increment kv data: " + e); + return null; + } + } + + // batch + public boolean batch() { + if (obTableClient == null) { + System.out.println("kv client is null"); + return false; + } + try { + TableBatchOps batchOps = obTableClient.batch(tableName); + batchOps.get((String)"key1", new String[]{"val"}); + batchOps.insert((String)"key5", new String[]{"val"}, new Object[]{"insert key5"}); + batchOps.update((String)"key5", new String[]{"val"}, new Object[]{"update key5"}); + batchOps.insertOrUpdate((String)"key6", new String[]{"val"}, new Object[]{"insertOrUpdate key6"}); + batchOps.append((String)"key3", new String[]{"val"}, new Object[]{"_append"}, (boolean)true); + List retObj = batchOps.execute(); + if (retObj.size() != 5) { + System.out.println("batch Ops error"); + return false; + } + System.out.println("batch Ops success."); + return true; + } catch (Exception e) { + System.out.println("fail to execute batch ops: " + e); + return false; + } + } +} diff --git a/example/simple-kv-demo/src/main/java/com/oceanbase/example/LoggerFactory.java b/example/simple-kv-demo/src/main/java/com/oceanbase/example/LoggerFactory.java new file mode 100644 index 00000000..43d4a80c --- /dev/null +++ b/example/simple-kv-demo/src/main/java/com/oceanbase/example/LoggerFactory.java @@ -0,0 +1,44 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.oceanbase.example; + + +import com.alipay.sofa.common.code.LogCode2Description; +import com.alipay.sofa.common.log.LoggerSpaceManager; +import org.slf4j.Logger; + + +public class LoggerFactory { + public static final String OCEANBASE_TABLE_TEST_LOGGER_SPACE = "oceanbase-table-test"; + public static LogCode2Description LCD = LogCode2Description.create(OCEANBASE_TABLE_TEST_LOGGER_SPACE); + + + public static Logger getLogger(String name) { + if (name == null || name.isEmpty()) { + return null; + } + return LoggerSpaceManager.getLoggerBySpace(name, OCEANBASE_TABLE_TEST_LOGGER_SPACE); + } + + public static Logger getLogger(Class klass) { + if (klass == null) { + return null; + } + return getLogger(klass.getCanonicalName()); + } +} diff --git a/example/simple-kv-demo/src/main/java/com/oceanbase/example/SimpleKVDemo.java b/example/simple-kv-demo/src/main/java/com/oceanbase/example/SimpleKVDemo.java new file mode 100644 index 00000000..afc45017 --- /dev/null +++ b/example/simple-kv-demo/src/main/java/com/oceanbase/example/SimpleKVDemo.java @@ -0,0 +1,106 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.oceanbase.example; + +public class SimpleKVDemo { + + public static void main(String[] args) { + KVClient kvClient = new KVClient(); + try { + if (!kvClient.initial()) { + return; + } + // del + kvClient.del((String)"key1"); + kvClient.del((String)"key2"); + kvClient.del((String)"key3"); + + // get after del + System.out.println("get after delete ..."); + System.out.println("key1 -> " + kvClient.get("key1")); + System.out.println("key2 -> " + kvClient.get("key2")); + System.out.println("key3 -> " + kvClient.get("key3")); + + // insert + kvClient.put((String)"key1", "val1"); + kvClient.put((String)"key2", "val2"); + kvClient.put((String)"key3", "val3"); + + // get after insert + System.out.println("get after insert ..."); + System.out.println("key1 -> " + kvClient.get("key1")); + System.out.println("key2 -> " + kvClient.get("key2")); + System.out.println("key3 -> " + kvClient.get("key3")); + + // update + boolean updateRet = kvClient.update((String)"key1", (String)"update val1"); + if (!updateRet) { + System.out.println("update error"); + } else { + // get after update + System.out.println("key1 -> " + kvClient.get("key1")); + } + + // insert or update + // insert new row + boolean insertOrUpdateRet1 = kvClient.insertOrUpdate((String)"key4", (String)"insert val4"); + if (!insertOrUpdateRet1) { + System.out.println("insertOrUpdate error"); + } + // insert or update old row + boolean insertOrUpdateRet2 = kvClient.insertOrUpdate((String)"key3", (String)"update old val3"); + if (!insertOrUpdateRet2) { + System.out.println("insertOrUpdate error"); + } else { + // get after insertOrUpdate + System.out.println("get after insertOrUpdate ..."); + System.out.println("key3 -> " + kvClient.get("key3")); + System.out.println("key4 -> " + kvClient.get("key4")); + } + + // replace of row 2 + boolean replace_ret = kvClient.replace((String)"key2", (String)"replace val2"); + if (!replace_ret) { + System.out.println("replace_ret error"); + } else { + System.out.println("key2 -> " + kvClient.get("key2")); + } + + // append + String append_ret = kvClient.append((String)"key4", (String)"_append"); + if (append_ret == null) { + System.out.println("append error"); + } else { + System.out.println("get after append ..."); + System.out.println("key4 -> " + kvClient.get("key4")); + } + + // batch + boolean batchOps = kvClient.batch(); + if (!batchOps) { + System.out.println("batch ops error"); + } else { + System.out.println("get after batch ops ..."); + System.out.println("key5 -> " + kvClient.get("key5")); + System.out.println("key6 -> " + kvClient.get("key6")); + } + } finally { + kvClient.close(); + } + } +} diff --git a/example/simple-mutation/README.md b/example/simple-mutation/README.md new file mode 100644 index 00000000..9c2d37fc --- /dev/null +++ b/example/simple-mutation/README.md @@ -0,0 +1,4 @@ +obkv-table-client-java demo project. + +## Notice +* **ObTableFilter** is not available now, similarly, mutation with filter cannot be used since the observer has not supported related functions. \ No newline at end of file diff --git a/example/simple-mutation/pom.xml b/example/simple-mutation/pom.xml new file mode 100644 index 00000000..0a854999 --- /dev/null +++ b/example/simple-mutation/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + com.oceanbase.example + simple-mutation + 1.0-SNAPSHOT + + + + 8 + 8 + + + + + com.oceanbase + obkv-table-client + 1.2.10 + + + + + \ No newline at end of file diff --git a/example/simple-mutation/src/main/java/com/oceanbase/example/MutationUtil.java b/example/simple-mutation/src/main/java/com/oceanbase/example/MutationUtil.java new file mode 100644 index 00000000..20577506 --- /dev/null +++ b/example/simple-mutation/src/main/java/com/oceanbase/example/MutationUtil.java @@ -0,0 +1,71 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.oceanbase.example; + +import com.alipay.oceanbase.rpc.ObTableClient; + +import java.util.Map; +import java.util.List; + +/* table schema: +CREATE TABLE IF NOT EXISTS `kv_table` ( + `key` varchar(20) NOT NULL, + `val` varchar(20) DEFAULT NULL, + PRIMARY KEY (`key`) +); +*/ + +public class MutationUtil { + private ObTableClient obTableClient = null; + private String tableName = "kv_table"; + + public boolean initial() { + try { + obTableClient = new ObTableClient(); + obTableClient.setFullUserName("your user name"); // e.g. root@sys#ocp + obTableClient.setParamURL("your configurl + database=xxx"); // e.g. http://ip:port/services?Action=ObRootServiceInfo&ObRegion=ocp&database=test + obTableClient.setPassword("your user passwd"); + obTableClient.setSysUserName("your sys user"); // e.g. proxyro@sys + obTableClient.setSysPassword("your sys user passwd"); + obTableClient.init(); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("fail to init OBKV client, " + e); + obTableClient = null; + return false; + } + System.out.println("initial OBKV client success"); + return true; + } + + public void close() { + if (obTableClient == null) { + return; + } + try { + obTableClient.close(); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("fail to close OBKV client, " + e); + } + } + + public ObTableClient getClient() { + return this.obTableClient; + } +} diff --git a/example/simple-mutation/src/main/java/com/oceanbase/example/SimpleMutation.java b/example/simple-mutation/src/main/java/com/oceanbase/example/SimpleMutation.java new file mode 100644 index 00000000..9832122c --- /dev/null +++ b/example/simple-mutation/src/main/java/com/oceanbase/example/SimpleMutation.java @@ -0,0 +1,199 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.oceanbase.example; + +import com.alipay.oceanbase.rpc.Mutation; + +/* table schema: +CREATE TABLE IF NOT EXISTS `kv_table` ( + `key` varchar(20) NOT NULL, + `val` varchar(20) DEFAULT NULL, + PRIMARY KEY (`key`) +); + +// test for aggregation +CREATE TABLE IF NOT EXISTS `aggregation_table` ( + `key` int NOT NULL, + `val` int DEFAULT NULL, + PRIMARY KEY (`key`) +); + +// test for auto increment +CREATE TABLE IF NOT EXISTS `auto_inc_table` ( + `key` int NOT NULL, + `val` int DEFAULT NULL, + PRIMARY KEY (`key`) +); + +// test for check and insert +CREATE TABLE IF NOT EXISTS `mutation_table` ( + `key` int NOT NULL, + `val` int DEFAULT NULL, + PRIMARY KEY (`key`) +); + +*/ + +public class SimpleMutation { + + public static void main(String[] args) { + MutationUtil mutationUtil = new MutationUtil(); + ObTableClient tableClient = null; + try { + if (!mutationUtil.initial()) { + return; + } + tableClient = mutationUtil.getClient(); + + // ColumnValue -> key-value pair + ColumnValue columnValue = colval("key", "value"); + + // Row -> multi-ColumnValue (row) + Row row = row(columnValue, new ColumnValue("other_key", "other_value")); + + // insert + Insert insert = tableClient.insert("kv_table") + .setRowKey(row(colVal("key", "key_0"))) + .addMutateColVal(colVal("val", "val_0")); + MutationResult result = insert.execute(); + System.out.println("insert " + (result.getAffectedRows()) + " rows"); + + // update + Update update = tableClient.update("kv_table") + .setRowKey(row(colVal("key", "key_0"))) + .addMutateColVal(colVal("val", "val_1")) + .setFilter(compareVal(ObCompareOp.EQ, "val", "val_0")); // not available, coming soon + MutationResult result = update.execute(); + System.out.println("update " + result.getAffectedRows() + " rows"); + + // delete + Delete delete = tableClient.delete("kv_table") + .setRowKey(row(colVal("key", "key_0"))) + .setFilter(compareVal(ObCompareOp.EQ, "val", "val_0")); // not available, coming soon + MutationResult result = delete.execute(); + System.out.println("delete " + (result.getAffectedRows()) + " rows"); + + // insertOrUpdate + InsertOrUpdate insertOrUpdate = tableClient.insertOrUpdate("kv_table") + .setRowKey(row(colVal("key", "key_0"))) + .addMutateColVal(colVal("val", "val_0")); + MutationResult result = insertOrUpdate.execute(); + System.out.println("insertOrUpdate " + (result.getAffectedRows()) + " rows"); + + // replace + Replace replace = tableClient.replace("kv_table") + .setRowKey(row(colVal("key", "key_0"))) + .addMutateColVal(colVal("val", "val_0")); + MutationResult result = replace.execute(); + System.out.println("replace " + (result.getAffectedRows()) + " rows"); + + // increment + Increment increment = tableClient.increment("kv_table") + .setRowKey(row(colVal("key", "key_0"))) + .addMutateColVal(colVal("val", "val_1")) + .setFilter(compareVal(ObCompareOp.EQ, "val", "val_0")); // not available, coming soon + MutationResult result = increment.execute(); + System.out.println("increment " + result.getAffectedRows() + " rows"); + + // append + Append append = tableClient.append("kv_table") + .setRowKey(row(colVal("key", "key_0"))) + .addMutateColVal(colVal("val", "val_1")) + .setFilter(compareVal(ObCompareOp.EQ, "val", "val_0")); // not available, coming soon + MutationResult result = append.execute(); + System.out.println("append " + result.getAffectedRows() + " rows"); + + + // batch operation + // construct single mutation + // filter could not be used in all operation in batchOperation + Insert insert_0 = insert().setRowKey(colVal("key", "key_1")) + .addMutateColVal(colVal("val", "val_1")) + + Insert insert_1 = insert().setRowKey(colVal("key", "key_1")) + .addMutateColVal(colVal("val", "val_1")) + + Update update_0 = update().setRowKey(colVal("key", "key_1")) + .addMutateColVal(colVal("val", "val_2")) + + // construct batch operation + BatchMutationResult batchResult = client.batchMutation("kv_table") + .addMutation(insert_0) + .addMutation(insert_1, update_0) + .execute(); + + // print output + for (int idx : batchResult.size()) { + System.out.println(String.format("the %dth mutation affect %d rows", idx, + batchResult.get(idx).getAffectedRows())); + } + + /* + * notice. these are only demo for new operations + */ + + // aggregation + // construct aggregation + ObTableAggregation obtableAggregation = client.aggregate("aggregation_table"); + + // add aggregation operation + obtableAggregation.max("c2"); + + // execute aggregation + ObTableAggregationResult obtableAggregationResult = obtableAggregation.execute(); + + // get aggregation result + System.out.println(obtableAggregationResult.get("max(c2)")); + + // auto increment + // if rowkey is auto increment + // use auto increment value, should fill 0 + // the value of new row on c1 will be 1 + client.insert("auto_inc_table", new Object[] { 0 }, new String[] {"c2"}, + new Object[] { 1 }); + + // assign the specific value on auto increment column + // the value of new row on c1 will be 100 + client.insert("auto_inc_table", new Object[] { 100 }, new String[] {"c2"}, + new Object[] { 1 }); + + //assign the specific value will refresh the auto increment value + // the value of new row on c1 will be 101 + client.insert("auto_inc_table", new Object[] { 0 }, new String[] {"c2"}, + new Object[] { 1 }); + + // check and insert + // use setRowkey to specify the key of the new row + // use addScanRange to add a rowkey range where to check filter + // keep the new row and the range of filter in the same partition + // should set range and filter both + // suppose old row (1, 1) exists + // satisfy the filter, insert new row (2, 2) + ObTableValueFilter filter = compareVal(ObCompareOp.EQ, "c2", 1); + MutationResult insertResult = client.insert("mutation_table") + .setRowKey(colVal("c1", 2)).setFilter(filter) + .addScanRange(new Object[] { 1 }, new Object[] { 200 }) + .addMutateRow(row(colVal("c2", 2)).execute(); + + } finally { + if (tableClient != null) { + tableClient.close(); + } + } + } +} diff --git a/example/simple-table-demo/README.md b/example/simple-table-demo/README.md new file mode 100644 index 00000000..e69de29b diff --git a/example/simple-table-demo/pom.xml b/example/simple-table-demo/pom.xml new file mode 100644 index 00000000..462dd0d4 --- /dev/null +++ b/example/simple-table-demo/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + com.oceanbase.example + simple-table-demo + 1.0-SNAPSHOT + + + + 8 + 8 + + + + + com.oceanbase + obkv-table-client + 1.2.10 + + + + + diff --git a/example/simple-table-demo/src/main/java/com/oceanbase/example/LoggerFactory.java b/example/simple-table-demo/src/main/java/com/oceanbase/example/LoggerFactory.java new file mode 100644 index 00000000..43d4a80c --- /dev/null +++ b/example/simple-table-demo/src/main/java/com/oceanbase/example/LoggerFactory.java @@ -0,0 +1,44 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.oceanbase.example; + + +import com.alipay.sofa.common.code.LogCode2Description; +import com.alipay.sofa.common.log.LoggerSpaceManager; +import org.slf4j.Logger; + + +public class LoggerFactory { + public static final String OCEANBASE_TABLE_TEST_LOGGER_SPACE = "oceanbase-table-test"; + public static LogCode2Description LCD = LogCode2Description.create(OCEANBASE_TABLE_TEST_LOGGER_SPACE); + + + public static Logger getLogger(String name) { + if (name == null || name.isEmpty()) { + return null; + } + return LoggerSpaceManager.getLoggerBySpace(name, OCEANBASE_TABLE_TEST_LOGGER_SPACE); + } + + public static Logger getLogger(Class klass) { + if (klass == null) { + return null; + } + return getLogger(klass.getCanonicalName()); + } +} diff --git a/example/simple-table-demo/src/main/java/com/oceanbase/example/ObTableDirectLoadExample.java b/example/simple-table-demo/src/main/java/com/oceanbase/example/ObTableDirectLoadExample.java new file mode 100644 index 00000000..4a781b51 --- /dev/null +++ b/example/simple-table-demo/src/main/java/com/oceanbase/example/ObTableDirectLoadExample.java @@ -0,0 +1,281 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.oceanbase.example; + +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObLoadDupActionType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableLoadClientStatus; +import com.alipay.oceanbase.rpc.table.ObTable; +import com.alipay.oceanbase.rpc.table.ObTableDirectLoad; +import com.alipay.oceanbase.rpc.table.ObDirectLoadBucket; +import com.alipay.oceanbase.rpc.table.ObDirectLoadObjRow; +import com.alipay.oceanbase.rpc.table.ObDirectLoadParameter; +import com.alipay.oceanbase.rpc.util.ObBytesString; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; + +public class ObTableDirectLoadExample { + + private static String host = "0.0.0.0"; + private static int port = 0; + + private static String tenantName = "mysql"; + private static String userName = "root"; + private static String password = ""; + private static String dbName = "test"; + private static String tableName = "lineitem"; + + // parameters of direct load + private static int parallel = 2; // Determines the number of server worker threads + private static long maxErrorRowCount = 0; + private static ObLoadDupActionType dupAction = ObLoadDupActionType.REPLACE; + private static long timeout = 1000L * 1000 * 1000; // The overall timeout of the direct load task + private static long heartBeatTimeout = 30L * 1000 * 1000; // The overall heartbeat timeout of the direct load task + + // load data from csv file + private static String CSVFilePath = "xxx.csv"; // The path of the csv file + private static byte lineTermDelimByte = '\n'; // The line separator + private static byte fieldTermDelimByte = '|'; // The field separator + + // statistics parameter + private static long totalRow = 0; + private static long readFileTime = 0; + private static long createBucketTime = 0; + + private static long printProgressPerRows = 10 * 10000; + + /* + * Create a ObObj value of ObBytesString. + */ + private static ObObj createObj(ObBytesString value) { + ObObj obj = new ObObj(); + obj.setMeta(ObObjType.defaultObjMeta(value.bytes)); + obj.setValue(value.bytes); + return obj; + } + + /* + * Split string and create a obj row. + */ + private static ObDirectLoadObjRow createObjRow(ObBytesString rowString) { + ObBytesString[] fieldsString = rowString.split(fieldTermDelimByte); + ObObj[] objs = new ObObj[fieldsString.length]; + for (int i = 0; i < fieldsString.length; ++i) { + objs[i] = createObj(fieldsString[i]); + } + return new ObDirectLoadObjRow(objs); + } + + /** + * Create a bucket store a bath rows data. + */ + private static ObDirectLoadBucket createBucket(ObBytesString[] rowsString, int limit) { + if (rowsString == null) + throw new NullPointerException(); + if (rowsString.length == 0 || limit > rowsString.length) + throw new IllegalArgumentException(String.format( + "invalid args, rowsString.length(%d), limit(%d)", rowsString.length, limit)); + long startTime = System.currentTimeMillis(); + ObDirectLoadBucket bucket = new ObDirectLoadBucket(); + for (int i = 0; i < limit; ++i) { + bucket.addRow(createObjRow(rowsString[i])); + } + long endTime = System.currentTimeMillis(); + createBucketTime += (endTime - startTime); + return bucket; + } + + /** + * Read row from csv file. + */ + public static class FileReader { + private RandomAccessFile raf; + private byte[] buffer; + private int readIndex = 0; + private int writeIndex = 0; + + public FileReader(String filePath) throws FileNotFoundException { + raf = new RandomAccessFile(filePath, "r"); + buffer = new byte[2 * 1024 * 1024]; + } + + public void close() throws IOException { + raf.close(); + } + + private void squashBuffer() { + if (readIndex > 0) { + System.arraycopy(buffer, readIndex, buffer, 0, writeIndex - readIndex); + writeIndex -= readIndex; + readIndex = 0; + } + } + + /* + * Read data from csv file to fill buffer. + */ + private int fillBuffer() throws IOException { + squashBuffer(); + long startTime = System.currentTimeMillis(); + int readLen = raf.read(buffer, writeIndex, buffer.length - writeIndex); + long endTime = System.currentTimeMillis(); + readFileTime += (endTime - startTime); + if (readLen != -1) { + writeIndex += readLen; + } + return readLen; + } + + /* + * Read a row of data from buffer. + */ + private ObBytesString readLineFromBuffer() { + ObBytesString line = null; + for (int i = readIndex; i < writeIndex; ++i) { + if (buffer[i] == lineTermDelimByte) { + byte[] lineBytes = new byte[i - readIndex]; + System.arraycopy(buffer, readIndex, lineBytes, 0, lineBytes.length); + line = new ObBytesString(lineBytes); + readIndex = i + 1; + break; + } + } + return line; + } + + /* + * Read a row of data. + */ + public ObBytesString readLine() throws IOException { + ObBytesString line = readLineFromBuffer(); + if (line == null && fillBuffer() != -1) { + line = readLineFromBuffer(); + } + return line; + } + + /** + * Read several lines of data + */ + public int readLines(ObBytesString[] lines) throws IOException { + int count = 0; + for (int i = 0; i < lines.length; ++i) { + ObBytesString line = readLine(); + if (line != null) { + lines[count++] = line; + } else { + break; + } + } + return count; + } + } + + private static void printStatisticsInfo() { + System.out.println(String.format("当前导入总行数：%d, 读文件耗时：%dms, 构造bucket耗时：%dms", totalRow, + readFileTime, createBucketTime)); + } + + private static void printFinalStatisticsInfo() { + System.out.println(String.format("导入总行数：%d, 读文件耗时：%dms, 构造bucket耗时：%dms", totalRow, + readFileTime, createBucketTime)); + } + + public static void main(String[] args) { + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + printFinalStatisticsInfo(); + } + }); + + ObTable table = null; + try { + table = new ObTable.Builder(host, port).setLoginInfo(tenantName, userName, password, + dbName).build(); + } catch (Exception e) { + throw new RuntimeException(e); + } + // create a direct load object for a table + ObDirectLoadParameter parameter = new ObDirectLoadParameter(); + parameter.setParallel(parallel); + parameter.setMaxErrorRowCount(maxErrorRowCount); + parameter.setDupAction(dupAction); + parameter.setTimeout(timeout); + parameter.setHeartBeatTimeout(heartBeatTimeout); + ObTableDirectLoad directLoad = new ObTableDirectLoad(table, tableName, parameter,true); + + try { + // begin direct load for a table + // is successful, table will be locked + directLoad.begin(); + // specify the source data segment id to start a trans for load the source segment data + FileReader fileReader = new FileReader(CSVFilePath); + ObBytesString[] rowsString = new ObBytesString[100]; + int rowCount = 0; + while ((rowCount = fileReader.readLines(rowsString)) > 0) { + // construct a bucket + ObDirectLoadBucket bucket = createBucket(rowsString, rowCount); + // send the bucket to observer + directLoad.insert(bucket); + // update statistics info + long prev = totalRow / printProgressPerRows; + totalRow += bucket.getRowCount(); + long now = totalRow / printProgressPerRows; + if (now > prev) { + printStatisticsInfo(); + } + } + // commit direct load + directLoad.commit(); + + // check commit complete + while (true) { + ObTableLoadClientStatus status = directLoad.getStatus(); + if (directLoad.isCommitting()) { + // wait + } else if (directLoad.isCommit()) { + System.out.println("commit"); + break; + } else if (directLoad.isAbort()) { + System.out.println("abort"); + break; + } else { + System.out.println("error status:" + status.toString()); + directLoad.abort(); + break; + } + Thread.sleep(500); + } + } catch (Exception e) { + e.printStackTrace(); + try { + // unlock table and release direct load objects + directLoad.abort(); + } catch (Exception e2) { + throw new RuntimeException(e2); + } + } finally { + table.close(); + } + + } + +} diff --git a/example/simple-table-demo/src/main/java/com/oceanbase/example/SimpleTableDemo.java b/example/simple-table-demo/src/main/java/com/oceanbase/example/SimpleTableDemo.java new file mode 100644 index 00000000..cff3e9fa --- /dev/null +++ b/example/simple-table-demo/src/main/java/com/oceanbase/example/SimpleTableDemo.java @@ -0,0 +1,124 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.oceanbase.example; + +public class SimpleTableDemo { + + public static void main(String[] args) { + TableClient tableClient = new TableClient(); + try { + if (!tableClient.initial()) { + return; + } + // del + tableClient.del((long)1); + tableClient.del((long)2); + tableClient.del((long)3); + + // get after del + System.out.println("get after delete ..."); + System.out.println("key_1 -> " + tableClient.get((long)1)); + System.out.println("key_2 -> " + tableClient.get((long)2)); + System.out.println("key_3 -> " + tableClient.get((long)3)); + + // insert + tableClient.put((long)1, new Object[]{(int)11, "c3_1"}); + tableClient.put((long)2, new Object[]{(int)22, "c3_2"}); + tableClient.put((long)3, new Object[]{(int)33, "c3_3"}); + + // get after insert + System.out.println("get after insert ..."); + System.out.println("key_1 -> " + "c2:" + tableClient.get((long)1)[0] + ", c3:" + tableClient.get((long)1)[1]); + System.out.println("key_2 -> " + "c2:" + tableClient.get((long)2)[0] + ", c3:" + tableClient.get((long)2)[1]); + System.out.println("key_3 -> " + "c2:" + tableClient.get((long)3)[0] + ", c3:" + tableClient.get((long)3)[1]); + + // update + boolean updateRet = tableClient.update((long)1, new Object[]{(int)111, "update c3_1"}); + if (!updateRet) { + System.out.println("update error"); + } else { + // get after update + System.out.println("key_1 -> " + "c2:" + tableClient.get((long)1)[0] + ", c3:" + tableClient.get((long)1)[1]); + } + + // insert or update + // insert new row + boolean insertOrUpdateRet1 = tableClient.insertOrUpdate((long)4, new Object[]{(int)44, "insert c3_4"}); + if (!insertOrUpdateRet1) { + System.out.println("insertOrUpdate error"); + } + // insert or update old row + boolean insertOrUpdateRet2 = tableClient.insertOrUpdate((long)3, new Object[]{(int)333, "update old c3_3"}); + if (!insertOrUpdateRet2) { + System.out.println("insertOrUpdate error"); + } else { + // get after insertOrUpdate + System.out.println("get after insertOrUpdate ..."); + System.out.println("key_3 -> " + "c2:" + tableClient.get((long)3)[0] + ", c3:" + tableClient.get((long)3)[1]); + System.out.println("key_4 -> " + "c2:" + tableClient.get((long)4)[0] + ", c3:" + tableClient.get((long)4)[1]); + } + + // replace of row 2 + boolean replace_ret = tableClient.replace((long)2, new Object[]{(int)222, "replace c3_2"}); + if (!replace_ret) { + System.out.println("replace_ret error"); + } else { + System.out.println("key_2 -> " + "c2:" + tableClient.get((long)2)[0] + ", c3:" + tableClient.get((long)2)[1]); + } + + // increment of row 4 c2 + boolean inc_result = tableClient.increment((long)4, (int)1); + if (!inc_result) { + System.out.println("increment error"); + } else { + System.out.println("get after increment ..."); + System.out.println("key_4 -> " + "c2:" + tableClient.get((long)4)[0] + ", c3:" + tableClient.get((long)4)[1]); + } + + // append + String append_ret = tableClient.append((long)4, (String)"_append"); + if (append_ret == null) { + System.out.println("append error"); + } else { + System.out.println("get after append ..."); + System.out.println("key_4 -> " + "c2:" + tableClient.get((long)4)[0] + ", c3:" + tableClient.get((long)4)[1]); + } + + // batch + boolean batchOps = tableClient.batch(); + if (!batchOps) { + System.out.println("batch ops error"); + } else { + System.out.println("get after batch ops ..."); + System.out.println("key_5 -> " + "c2:" + tableClient.get((long)5)[0] + ", c3:" + tableClient.get((long)5)[1]); + System.out.println("key_6 -> " + "c2:" + tableClient.get((long)6)[0] + ", c3:" + tableClient.get((long)6)[1]); + } + + // scan + boolean queryRet = tableClient.query(); + if (!queryRet) { + System.out.println("query table error"); + } else { + System.out.println("query table success."); + } + + } finally { + tableClient.close(); + } + } +} diff --git a/example/simple-table-demo/src/main/java/com/oceanbase/example/TableClient.java b/example/simple-table-demo/src/main/java/com/oceanbase/example/TableClient.java new file mode 100644 index 00000000..2bf7fb3a --- /dev/null +++ b/example/simple-table-demo/src/main/java/com/oceanbase/example/TableClient.java @@ -0,0 +1,296 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.oceanbase.example; + + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.table.api.TableBatchOps; +import com.alipay.oceanbase.rpc.table.api.TableQuery; + +import java.util.Map; +import java.util.List; + + +/* +* 此demo只用作指导使用tableapi，测试前请先使用下列schema在SQL中建立table +* */ + +/* 测试表schema: +CREATE TABLE IF NOT EXISTS `test_table` ( + `c1` bigint NOT NULL, + `c2` int NOT NULL, + `c3` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) +); +*/ + +public class TableClient { + private ObTableClient obTableClient = null; + private String tableName = "test_table"; + + public boolean initial() { + try { + obTableClient = new ObTableClient(); + obTableClient.setFullUserName("your user name"); // e.g. root@sys#ocp + obTableClient.setParamURL("your configurl + database=xxx"); // e.g. http://ip:port/services?Action=ObRootServiceInfo&ObRegion=ocp&database=test + obTableClient.setPassword("your user passwd"); + obTableClient.setSysUserName("your sys user"); // e.g. proxyro@sys + obTableClient.setSysPassword("your sys user passwd"); + + obTableClient.init(); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("fail to init table client, " + e); + obTableClient = null; + return false; + } + System.out.println("initial table client success"); + return true; + } + + public void close() { + if (obTableClient == null) { + return; + } + try { + obTableClient.close(); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("fail to close table client, " + e); + } + } + + public boolean put(Object key, Object[] values) { + if (obTableClient == null) { + System.out.println("table client is null"); + return false; + } + try { + long rows = obTableClient.insert(tableName, key, new String[]{"c2", "c3"}, values); + if (rows != 1) { + System.out.println("fail to put table data"); + return false; + } + } catch (Exception e) { + e.printStackTrace(); + System.out.println("fail to put table data, " + e); + return false; + } + System.out.println("put table data success"); + return true; + } + + // return all column of row + public Object[] get(Object key) { + Object[] colResult = new Object[2]; + if (obTableClient == null) { + System.out.println("table client is null"); + return null; + } + try { + Map result = obTableClient.get(tableName, key, new String[]{"c2","c3"}); + colResult[0] = (int)result.get("c2"); + colResult[1] = (String)result.get("c3"); + return colResult; + } catch (Exception e) { + System.out.println("fail to get table data"); + return null; + } + } + + public boolean update(Object key, Object[] val) { + if (obTableClient == null) { + System.out.println("table client is null"); + return false; + } + try { + long rows = obTableClient.update(tableName, key, new String[]{"c2","c3"}, val); + if (rows != 1) { + System.out.println("fail to put table data"); + return false; + } + } catch (Exception e) { + System.out.println("fail to get table data"); + return false; + } + System.out.println("update table data success"); + return true; + } + // del + public boolean del(Object key) { + if (obTableClient == null) { + System.out.println("table client is null"); + return false; + } + try { + long rows = obTableClient.delete(tableName, key); + if (rows != 1) { + System.out.println("del table data success, row = 0"); + return true; + } + } catch (Exception e) { + System.out.println("fail to del table data"); + return false; + } + System.out.println("del table data success, row = 1"); + return true; + } + + // insert or update + public boolean insertOrUpdate(Object key, Object[] val) { + if (obTableClient == null) { + System.out.println("table client is null"); + return false; + } + try { + long rows = obTableClient.insertOrUpdate(tableName, key, new String[]{"c2", "c3"}, val); + if (rows != 1) { + System.out.println("insertOrUpdate table data success, row = 0"); + return true; + } + } catch (Exception e) { + System.out.println("fail to insertOrUpdate table data" + e); + return false; + } + System.out.println("insertOrUpdate table data success, row = 1"); + return true; + } + + // replace + public boolean replace(Object key, Object[] replace_val) { + if (obTableClient == null) { + System.out.println("table client is null"); + return false; + } + try { + long rows = obTableClient.replace(tableName, key, new String[]{"c2","c3"}, replace_val); + if (rows != 1) { + System.out.println("replace table data success, row = 0"); + return true; + } + } catch (Exception e) { + System.out.println("fail to replace table data" + e); + return false; + } + System.out.println("replace table data success, row = 1"); + return true; + } + + // increment + public boolean increment(Object key, int inc_val) { + if (obTableClient == null) { + System.out.println("table client is null"); + return false; + } + try { + // get old value first + Map result; + int oldVal, newVal; + result = obTableClient.get(tableName, key, new String[]{"c2","c3"}); + if (result == null) { + return false; + } + oldVal = (int)result.get("c2"); + // increment and get new value; + result = obTableClient.increment(tableName, key, new String[]{"c2"}, new Object[]{inc_val}, (boolean)true); + if (result == null) { + return false; + } + newVal = (int)result.get("c2"); + if (oldVal + inc_val != newVal) { + return false; + } + return true; + } catch (Exception e) { + System.out.println("fail to increment table data: " + e); + return false; + } + } + + // append + public String append(Object key, String append_str) { + if (obTableClient == null) { + System.out.println("table client is null"); + return null; + } + try { + Map result = obTableClient.append(tableName, key, new String[]{"c3"}, new Object[]{append_str}, (boolean)true); + return (String)result.get("c3"); + } catch (Exception e) { + System.out.println("fail to increment table data: " + e); + return null; + } + } + + // scan + public boolean query() { + if (obTableClient == null) { + System.out.println("table client is null"); + return false; + } + try { + TableQuery tblQuery = obTableClient.query(tableName); + QueryResultSet resultSet = tblQuery.select("c1","c2","c3").primaryIndex() + .addScanRange((long)1, (long)5).execute(); + // print scan result + System.out.println("query result size: " + resultSet.cacheSize()); + int size = resultSet.cacheSize(); + for (int i = 0; i < size; i++) { + resultSet.next(); + Map row = resultSet.getRow(); // get one row + for (Map.Entry entry : row.entrySet()) { + System.out.println(entry.getKey() + "," + entry.getValue()); + } + } + } catch (Exception e) { + System.out.println("fail to increment table data: " + e); + return false; + } + return true; + } + + // batch + public boolean batch() { + if (obTableClient == null) { + System.out.println("table client is null"); + return false; + } + try { + TableBatchOps batchOps = obTableClient.batch(tableName); + batchOps.get((long)1, new String[]{"c2", "c3"}); + batchOps.insert((long)5, new String[]{"c2", "c3"}, new Object[]{(int)5, "batch new c3_5"}); + batchOps.update((long)5, new String[]{"c2"}, new Object[]{(int)55}); + batchOps.insertOrUpdate((long)6, new String[]{"c2","c3"}, new Object[]{(int)6, "batch new c3_6"}); + batchOps.increment((long)6, new String[]{"c2"}, new Object[]{(int)1}, (boolean)true); + batchOps.append((long)6, new String[]{"c3"}, new Object[]{"_append"}, (boolean)true); + List retObj = batchOps.execute(); + if (retObj.size() != 6) { + System.out.println("batch Ops error"); + return false; + } + System.out.println("batch Ops success."); + return true; + } catch (Exception e) { + System.out.println("fail to execute batch ops: " + e); + return false; + } + } + + // todo: others op. +} diff --git a/pom.xml b/pom.xml index 75a8c2de..119dfef7 100644 --- a/pom.xml +++ b/pom.xml @@ -4,53 +4,107 @@ com.oceanbase obkv-table-client - 0.1.0 + 1.2.13-SNAPSHOT - OBKV Table Client Framework + ${project.groupId}:${project.artifactId} + OceanBase JavaClient for TableApi + https://github.com/oceanbase/obkv-table-client-java + + + + Mulan Permissive Software License，Version 2 + http://license.coscl.org.cn/MulanPSL2/ + + + + + + oceanbaseObkvDevelopers + oceanbase obkv developers + OceanBase + + + + + scm:git:https://github.com/oceanbase/obkv-table-client-java.git + HEAD + https://github.com/oceanbase/obkv-table-client-java + + + + + obkv-table-client-staging + ${project.name} + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + obkv-table-client-snapshot + ${project.name} + https://s01.oss.sonatype.org/content/repositories/snapshots/ + + - 1.6 - 1.6 + 1.8 + 1.8 ${project.encoding} UTF-8 + + + + org.testcontainers + testcontainers-bom + 1.15.3 + pom + import + + + + com.alipay.sofa.common sofa-common-tools - 1.3.3 + 1.3.12 com.alipay.sofa bolt - 1.5.2 + 1.5.10 + + + netty-all + io.netty + + io.netty netty-all - 4.1.13.Final + 4.1.100.Final com.google.guava guava - 30.1.1-jre + 31.1-jre org.springframework spring-core - 4.3.4.RELEASE + 5.2.25.RELEASE test org.springframework spring-beans - 4.3.4.RELEASE + 5.2.25.RELEASE test @@ -63,36 +117,28 @@ com.alibaba fastjson - 1.2.69_sec12 + 1.2.83 mysql mysql-connector-java - 5.1.30 + 8.0.28 org.slf4j slf4j-api - 1.7.21 + 1.7.36 junit junit - 4.12 + 4.13.2 test - - com.yahoo.ycsb - core - 0.12.0 - system - ${project.basedir}/core-0.12.0.jar - - org.powermock powermock-api-mockito2 @@ -106,6 +152,19 @@ 2.0.9 test + + + ch.qos.logback + logback-classic + 1.2.12 + test + + + + org.testcontainers + jdbc + test + @@ -191,6 +250,18 @@ true + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + org.apache.maven.plugins maven-jar-plugin @@ -206,6 +277,20 @@ + + org.apache.maven.plugins + maven-gpg-plugin + 3.0.1 + + + sign-artifacts + verify + + sign + + + + com.googlecode.maven-java-formatter-plugin maven-java-formatter-plugin @@ -280,4 +365,4 @@ - + \ No newline at end of file diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java index 43561a71..80e11b0e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableBatchOps.java @@ -17,9 +17,7 @@ package com.alipay.oceanbase.rpc; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableBatchOperation; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableBatchOperationResult; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; import com.alipay.oceanbase.rpc.table.AbstractTableBatchOps; import com.alipay.oceanbase.rpc.table.ObTableClientBatchOpsImpl; @@ -40,7 +38,7 @@ public class ObClusterTableBatchOps extends AbstractTableBatchOps { this.tableBatchOps.setExecutorService(executorService); } - /** + /* * Get. */ @Override @@ -48,7 +46,7 @@ public void get(Object[] rowkeys, String[] columns) { tableBatchOps.get(rowkeys, columns); } - /** + /* * Update. */ @Override @@ -56,7 +54,7 @@ public void update(Object[] rowkeys, String[] columns, Object[] values) { tableBatchOps.update(rowkeys, columns, values); } - /** + /* * Delete. */ @Override @@ -64,7 +62,7 @@ public void delete(Object[] rowkeys) { tableBatchOps.delete(rowkeys); } - /** + /* * Insert. */ @Override @@ -72,7 +70,7 @@ public void insert(Object[] rowkeys, String[] columns, Object[] values) { tableBatchOps.insert(rowkeys, columns, values); } - /** + /* * Replace. */ @Override @@ -80,7 +78,7 @@ public void replace(Object[] rowkeys, String[] columns, Object[] values) { tableBatchOps.replace(rowkeys, columns, values); } - /** + /* * Insert or update. */ @Override @@ -88,7 +86,7 @@ public void insertOrUpdate(Object[] rowkeys, String[] columns, Object[] values) tableBatchOps.insertOrUpdate(rowkeys, columns, values); } - /** + /* * Increment. */ @Override @@ -96,7 +94,7 @@ public void increment(Object[] rowkeys, String[] columns, Object[] values, boole tableBatchOps.increment(rowkeys, columns, values, withResult); } - /** + /* * Append. */ @Override @@ -104,29 +102,48 @@ public void append(Object[] rowkeys, String[] columns, Object[] values, boolean tableBatchOps.append(rowkeys, columns, values, withResult); } - /** + /* + * Put. + */ + @Override + public void put(Object[] rowkeys, String[] columns, Object[] values) { + tableBatchOps.put(rowkeys, columns, values); + } + + /* * Execute. */ @Override public List execute() throws Exception { + preCheck(); return tableBatchOps.execute(); } - /** + /* + * Execute with result + */ + @Override + public List executeWithResult() throws Exception { + preCheck(); + return tableBatchOps.executeWithResult(); + } + + /* * Execute internal. */ public ObTableBatchOperationResult executeInternal() throws Exception { + preCheck(); return tableBatchOps.executeInternal(); } - /** + /* * clear batch operations */ public void clear() { tableBatchOps.clear(); } - /** + /* * Get ob table batch operation. */ @Override @@ -134,7 +151,7 @@ public ObTableBatchOperation getObTableBatchOperation() { return tableBatchOps.getObTableBatchOperation(); } - /** + /* * Get table name. */ @Override @@ -142,7 +159,7 @@ public String getTableName() { return tableBatchOps.getTableName(); } - /** + /* * Set entity type. */ @Override @@ -151,7 +168,7 @@ public void setEntityType(ObTableEntityType entityType) { tableBatchOps.setEntityType(entityType); } - /** + /* * Set atomic operation. */ @Override @@ -159,4 +176,26 @@ public void setAtomicOperation(boolean atomicOperation) { super.setAtomicOperation(atomicOperation); tableBatchOps.setAtomicOperation(atomicOperation); } + + @Override + public void setReturnOneResult(boolean returnOneResult) { + super.setReturnOneResult(returnOneResult); + tableBatchOps.setReturnOneResult(returnOneResult); + } + + void preCheck() { + List operations = this.tableBatchOps.getObTableBatchOperation() + .getTableOperations(); + if (operations.isEmpty()) { + throw new IllegalArgumentException("operations is empty"); + } + ObTableOperationType lastType = operations.get(0).getOperationType(); + if (returnOneResult + && !(this.tableBatchOps.getObTableBatchOperation().isSameType() && (lastType == ObTableOperationType.INSERT + || lastType == ObTableOperationType.PUT + || lastType == ObTableOperationType.REPLACE || lastType == ObTableOperationType.DEL))) { + throw new IllegalArgumentException( + "returnOneResult only support multi-insert/put/replace/del"); + } + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableQuery.java b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableQuery.java index fc9a996d..1192c873 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableQuery.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObClusterTableQuery.java @@ -17,24 +17,36 @@ package com.alipay.oceanbase.rpc; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation.ObTableAggregationType; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHTableFilter; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQuery; +import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.stream.QueryResultSet; import com.alipay.oceanbase.rpc.table.AbstractTableQuery; import com.alipay.oceanbase.rpc.table.ObTableClientQueryImpl; import com.alipay.oceanbase.rpc.table.api.TableQuery; +import java.util.List; + public class ObClusterTableQuery extends AbstractTableQuery { private final ObTableClientQueryImpl tableClientQuery; - ObClusterTableQuery(ObTableClientQueryImpl tableQuery) { + public ObClusterTableQuery(ObTableClientQueryImpl tableQuery) { this.tableClientQuery = tableQuery; } - /** + /* + * Add aggregation. + */ + public void addAggregation(ObTableAggregationType aggType, String aggColumn) { + this.tableClientQuery.addAggregation(aggType, aggColumn); + } + + /* * Get table name. */ @Override @@ -42,7 +54,7 @@ public String getTableName() { return tableClientQuery.getTableName(); } - /** + /* * Get ob table query. */ @Override @@ -50,7 +62,21 @@ public ObTableQuery getObTableQuery() { return tableClientQuery.getObTableQuery(); } - /** + /* + * Get select columns (used by BatchOperation) + */ + public List getSelectColumns() { + return tableClientQuery.getSelectColumns(); + } + + /* + * Get row key (used by BatchOperation) + */ + public Row getRowKey() { + return tableClientQuery.getRowKey(); + } + + /* * Execute. */ @Override @@ -58,14 +84,29 @@ public QueryResultSet execute() throws Exception { return tableClientQuery.execute(); } - /** + /* + * Execute. + */ + @Override + public QueryResultSet asyncExecute() throws Exception { + return tableClientQuery.asyncExecute(); + } + + /* * Execute internal. */ public ObTableClientQueryStreamResult executeInternal() throws Exception { return tableClientQuery.executeInternal(); } - /** + /* + * Async execute internal. + */ + public ObTableClientQueryAsyncStreamResult asyncExecuteInternal() throws Exception { + return tableClientQuery.asyncExecuteInternal(); + } + + /* * Select. */ @Override @@ -74,7 +115,7 @@ public TableQuery select(String... columns) { return this; } - /** + /* * 只有 limit query 需要，其他不需要 * @param keys * @return @@ -84,7 +125,15 @@ public TableQuery setKeys(String... keys) { throw new IllegalArgumentException("Not needed"); } - /** + /* + * set row key into query (only used bt BatchOperation) + */ + public TableQuery setRowKey(Row row) throws Exception { + tableClientQuery.setRowKey(row); + return this; + } + + /* * Limit. */ @Override @@ -164,6 +213,11 @@ public TableQuery setBatchSize(int batchSize) { return tableClientQuery.setBatchSize(batchSize); } + @Override + public TableQuery setMaxResultSize(long maxResultSize) { + return tableClientQuery.setMaxResultSize(maxResultSize); + } + /** * Clear. */ diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObGlobal.java b/src/main/java/com/alipay/oceanbase/rpc/ObGlobal.java new file mode 100644 index 00000000..4552e8a8 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/ObGlobal.java @@ -0,0 +1,95 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +public class ObGlobal { + static long OB_VSN_MAJOR_SHIFT = 32; + static long OB_VSN_MINOR_SHIFT = 16; + static long OB_VSN_MAJOR_PATCH_SHIFT = 8; + static long OB_VSN_MINOR_PATCH_SHIFT = 0; + static int OB_VSN_MAJOR_MASK = 0xffffffff; + static short OB_VSN_MINOR_MASK = (short) 0xffff; + static byte OB_VSN_MAJOR_PATCH_MASK = (byte) 0xff; + static byte OB_VSN_MINOR_PATCH_MASK = (byte) 0xff; + + public static long calcVersion(int major, short minor, byte major_patch, byte minor_patch) { + return ((long) major << OB_VSN_MAJOR_SHIFT) + ((long) minor << OB_VSN_MINOR_SHIFT) + + ((long) major_patch << OB_VSN_MAJOR_PATCH_SHIFT) + + ((long) minor_patch << OB_VSN_MINOR_PATCH_SHIFT); + } + + public static long calcVersion(long major, long minor, long major_patch, long minor_patch) { + return (major << OB_VSN_MAJOR_SHIFT) + (minor << OB_VSN_MINOR_SHIFT) + + (major_patch << OB_VSN_MAJOR_PATCH_SHIFT) + + (minor_patch << OB_VSN_MINOR_PATCH_SHIFT); + } + + public static int obVsnMajor() { + return getObVsnMajor(OB_VERSION); + } + + public static int getObVsnMajor(long version) { + return (int) ((version >> OB_VSN_MAJOR_SHIFT) & OB_VSN_MAJOR_MASK); + } + + public static short obVsnMinor() { + return getObVsnMinor(OB_VERSION); + } + + public static short getObVsnMinor(long version) { + return (short) ((version >> OB_VSN_MINOR_SHIFT) & OB_VSN_MINOR_MASK); + } + + public static byte obVsnMajorPatch() { + return getObVsnMajorPatch(OB_VERSION); + } + + public static byte getObVsnMajorPatch(long version) { + return (byte) ((version >> OB_VSN_MAJOR_PATCH_SHIFT) & OB_VSN_MAJOR_PATCH_MASK); + } + + public static byte obVsnMinorPatch() { + return getObVsnMinorPatch(OB_VERSION); + } + + public static byte getObVsnMinorPatch(long version) { + return (byte) ((version >> OB_VSN_MINOR_PATCH_SHIFT) & OB_VSN_MINOR_PATCH_MASK); + } + + public static String obVsnString() { + return String.format("%d.%d.%d.%d", obVsnMajor(), obVsnMinor(), obVsnMajorPatch(), + obVsnMinorPatch()); + } + + public static String getObVsnString(long version) { + return String.format("%d.%d.%d.%d", getObVsnMajor(version), getObVsnMinor(version), + getObVsnMajorPatch(version), getObVsnMinorPatch(version)); + } + + public static boolean isLsOpSupport() { + return OB_VERSION >= OB_VERSION_4_2_3_0 && OB_VERSION < OB_VERSION_4_3_0_0; + } + + public static final long OB_VERSION_4_2_1_0 = calcVersion(4, (short) 2, (byte) 1, (byte) 0); + + public static final long OB_VERSION_4_2_3_0 = calcVersion(4, (short) 2, (byte) 3, (byte) 0); + + public static final long OB_VERSION_4_3_0_0 = calcVersion(4, (short) 3, (byte) 0, (byte) 0); + + public static long OB_VERSION = calcVersion(0, (short) 0, (byte) 0, (byte) 0); +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index f7c436f4..31a7aa17 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -17,31 +17,31 @@ package com.alipay.oceanbase.rpc; +import com.alibaba.fastjson.JSON; +import com.alipay.oceanbase.rpc.checkandmutate.CheckAndInsUp; import com.alipay.oceanbase.rpc.constant.Constants; import com.alipay.oceanbase.rpc.exception.*; -import com.alipay.oceanbase.rpc.batch.QueryByBatch; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; import com.alipay.oceanbase.rpc.location.model.*; -import com.alipay.oceanbase.rpc.location.model.partition.ObPair; -import com.alipay.oceanbase.rpc.location.model.partition.ObPartitionLevel; +import com.alipay.oceanbase.rpc.location.model.partition.*; +import com.alipay.oceanbase.rpc.mutation.*; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation.ObTableAggregation; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutate; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateRequest; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObBorderFlag; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQuery; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryRequest; -import com.alipay.oceanbase.rpc.table.AbstractObTableClient; -import com.alipay.oceanbase.rpc.table.ObTable; -import com.alipay.oceanbase.rpc.table.ObTableClientBatchOpsImpl; -import com.alipay.oceanbase.rpc.table.ObTableClientQueryImpl; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; +import com.alipay.oceanbase.rpc.table.*; import com.alipay.oceanbase.rpc.table.api.TableBatchOps; import com.alipay.oceanbase.rpc.table.api.TableQuery; import com.alipay.oceanbase.rpc.threadlocal.ThreadLocalMap; -import com.alipay.oceanbase.rpc.util.StringUtil; -import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; -import com.alipay.oceanbase.rpc.util.ZoneUtil; +import com.alipay.oceanbase.rpc.util.*; import com.alipay.remoting.util.StringUtils; import org.slf4j.Logger; @@ -53,20 +53,17 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import static com.alipay.oceanbase.rpc.constant.Constants.ALL_DUMMY_TABLE; -import static com.alipay.oceanbase.rpc.constant.Constants.OCEANBASE_DATABASE; +import static com.alipay.oceanbase.rpc.constant.Constants.*; import static com.alipay.oceanbase.rpc.location.LocationUtil.*; import static com.alipay.oceanbase.rpc.location.model.ObServerRoute.STRONG_READ; import static com.alipay.oceanbase.rpc.location.model.TableEntry.HBASE_ROW_KEY_ELEMENT; -import static com.alipay.oceanbase.rpc.location.model.partition.ObPartConstants.PART_ID_BITNUM; -import static com.alipay.oceanbase.rpc.location.model.partition.ObPartConstants.PART_ID_SHIFT; +import static com.alipay.oceanbase.rpc.location.model.partition.ObPartIdCalculator.*; import static com.alipay.oceanbase.rpc.property.Property.*; import static com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType.*; -import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; +import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.*; public class ObTableClient extends AbstractObTableClient implements Lifecycle { - private static final Logger logger = TableClientLoggerFactory - .getLogger(ObTableClient.class); + private static final Logger logger = getLogger(ObTableClient.class); private static final String usernameSeparators = ":;-;."; @@ -74,7 +71,7 @@ public class ObTableClient extends AbstractObTableClient implements Lifecycle { 0); private String dataSourceName; private String paramURL; - /** + /* * user name * Standard format: user@tenant#cluster * NonStandard format: cluster:tenant:user @@ -86,7 +83,7 @@ public class ObTableClient extends AbstractObTableClient implements Lifecycle { private String password; private String database; - /** + /* * sys user auth to access meta table. */ private ObUserAuth sysUA = new ObUserAuth( @@ -95,12 +92,12 @@ public class ObTableClient extends AbstractObTableClient implements Lifecycle { private volatile OcpModel ocpModel = new OcpModel(); - /** + /* * ServerAddr(all) -> ObTableConnection */ private volatile ConcurrentHashMap tableRoster = null; - /** + /* * current tenant server address order by priority desc *

* be careful about concurrency when change the element @@ -109,18 +106,24 @@ public class ObTableClient extends AbstractObTableClient implements Lifecycle { private volatile RunningMode runningMode = RunningMode.NORMAL; - /** + /* * TableName -> TableEntry */ private Map tableLocations = new ConcurrentHashMap(); - /** + /* + * TableName -> ObIndexinfo + */ + private Map indexinfos = new ConcurrentHashMap(); + + private ConcurrentHashMap refreshIndexInfoLocks = new ConcurrentHashMap(); + + /* * TableName -> rowKey element */ private Map> tableRowKeyElement = new ConcurrentHashMap>(); - private boolean retryOnChangeMasterTimes = true; - /** + /* * TableName -> Failures/Lock */ private ConcurrentHashMap tableContinuousFailures = new ConcurrentHashMap(); @@ -139,7 +142,19 @@ public class ObTableClient extends AbstractObTableClient implements Lifecycle { private ObReadConsistency readConsistency = ObReadConsistency.STRONG; private ObRoutePolicy obRoutePolicy = ObRoutePolicy.IDC_ORDER; - /** + private boolean odpMode = false; + + private String odpAddr = "127.0.0.1"; + + private int odpPort = 2883; + + private ObTable odpTable = null; + // tableGroup <-> Table + private ConcurrentHashMap TableGroupCacheLocks = new ConcurrentHashMap(); + private ConcurrentHashMap TableGroupCache = new ConcurrentHashMap(); // tableGroup -> Table + private ConcurrentHashMap TableGroupInverted = new ConcurrentHashMap(); // Table -> tableGroup + + /* * Init. */ public void init() throws Exception { @@ -156,12 +171,17 @@ public void init() throws Exception { // 2. init metadata initMetadata(); initialized = true; + } catch (Throwable t) { + BOOT.warn("failed to init ObTableClient", t); + RUNTIME.warn("failed to init ObTableClient", t); + throw new RuntimeException(t); } finally { + BOOT.info("init ObTableClient successfully"); statusLock.unlock(); } } - /** + /* * Close. */ @Override @@ -183,7 +203,8 @@ public void close() throws Exception { entry.getValue().close(); } catch (Exception e) { // do not throw exception immediately - logger.error(LCD.convert("01-00004"), entry.getKey(), e); + BOOT.error(LCD.convert("01-00004"), entry.getKey(), e); + RUNTIME.error(LCD.convert("01-00004"), entry.getKey(), e); throwException = e; exceptionObServers.add(entry.getKey()); } @@ -201,33 +222,38 @@ public void close() throws Exception { throw new ObTableCloseException(sb.toString(), throwException); } } + if (odpTable != null) { + odpTable.close(); + } } finally { + BOOT.info("ObTableClient is closed"); statusLock.unlock(); } } - /** + /* * Check status. */ public void checkStatus() throws IllegalStateException { if (!initialized) { - throw new IllegalStateException("param url " + paramURL + "fullUserName" + fullUserName - + " is not initialized"); + throw new IllegalStateException("param url " + paramURL + "fullUserName " + + fullUserName + " is not initialized"); } if (closed) { - throw new IllegalStateException("param url " + paramURL + "fullUserName" + fullUserName - + " is closed"); + throw new IllegalStateException("param url " + paramURL + " fullUserName " + + fullUserName + " is closed"); } } private void initProperties() { + rpcConnectTimeout = parseToInt(RPC_CONNECT_TIMEOUT.getKey(), rpcConnectTimeout); // metadata.refresh.interval is preferred. - metadataRefreshInterval = parseToLong(METADATA_REFRESH_INTERNAL.getKey(), - metadataRefreshInterval); metadataRefreshInterval = parseToLong(METADATA_REFRESH_INTERVAL.getKey(), metadataRefreshInterval); + metadataRefreshInterval = parseToLong(METADATA_REFRESH_INTERNAL.getKey(), + metadataRefreshInterval); metadataRefreshLockTimeout = parseToLong(METADATA_REFRESH_LOCK_TIMEOUT.getKey(), metadataRefreshLockTimeout); @@ -242,26 +268,38 @@ private void initProperties() { rsListAcquireTryTimes); // rs.list.acquire.retry.interval is preferred. - rsListAcquireRetryInterval = parseToLong(RS_LIST_ACQUIRE_RETRY_INTERNAL.getKey(), - rsListAcquireRetryInterval); rsListAcquireRetryInterval = parseToLong(RS_LIST_ACQUIRE_RETRY_INTERVAL.getKey(), rsListAcquireRetryInterval); + rsListAcquireRetryInterval = parseToLong(RS_LIST_ACQUIRE_RETRY_INTERNAL.getKey(), + rsListAcquireRetryInterval); + + tableEntryAcquireConnectTimeout = parseToLong(TABLE_ENTRY_ACQUIRE_CONNECT_TIMEOUT.getKey(), + tableEntryAcquireConnectTimeout); + + tableEntryAcquireSocketTimeout = parseToLong(TABLE_ENTRY_ACQUIRE_SOCKET_TIMEOUT.getKey(), + tableEntryAcquireSocketTimeout); // table.entry.refresh.interval.base is preferred. - tableEntryRefreshIntervalBase = parseToLong(TABLE_ENTRY_REFRESH_INTERNAL_BASE.getKey(), - tableEntryRefreshIntervalBase); tableEntryRefreshIntervalBase = parseToLong(TABLE_ENTRY_REFRESH_INTERVAL_BASE.getKey(), tableEntryRefreshIntervalBase); + tableEntryRefreshIntervalBase = parseToLong(TABLE_ENTRY_REFRESH_INTERNAL_BASE.getKey(), + tableEntryRefreshIntervalBase); // table.entry.refresh.interval.ceiling is preferred. - tableEntryRefreshIntervalCeiling = parseToLong( - TABLE_ENTRY_REFRESH_INTERNAL_CEILING.getKey(), tableEntryRefreshIntervalCeiling); tableEntryRefreshIntervalCeiling = parseToLong( TABLE_ENTRY_REFRESH_INTERVAL_CEILING.getKey(), tableEntryRefreshIntervalCeiling); + tableEntryRefreshIntervalCeiling = parseToLong( + TABLE_ENTRY_REFRESH_INTERNAL_CEILING.getKey(), tableEntryRefreshIntervalCeiling); + + tableEntryRefreshIntervalWait = parseToBoolean(TABLE_ENTRY_REFRESH_INTERVAL_WAIT.getKey(), + tableEntryRefreshIntervalWait); tableEntryRefreshLockTimeout = parseToLong(TABLE_ENTRY_REFRESH_LOCK_TIMEOUT.getKey(), tableEntryRefreshLockTimeout); + tableEntryRefreshTryTimes = parseToInt(TABLE_ENTRY_REFRESH_TRY_TIMES.getKey(), + tableEntryRefreshTryTimes); + tableEntryRefreshContinuousFailureCeiling = parseToInt( TABLE_ENTRY_REFRESH_CONTINUOUS_FAILURE_CEILING.getKey(), tableEntryRefreshContinuousFailureCeiling); @@ -275,8 +313,7 @@ private void initProperties() { runtimeContinuousFailureCeiling = parseToInt(RUNTIME_CONTINUOUS_FAILURE_CEILING.getKey(), runtimeContinuousFailureCeiling); - int runtimeRetryTimes = parseToInt(RUNTIME_RETRY_TIMES.getKey(), this.runtimeRetryTimes); - this.runtimeRetryTimes = runtimeRetryTimes > 1 ? runtimeRetryTimes : this.runtimeRetryTimes; + this.runtimeRetryTimes = parseToInt(RUNTIME_RETRY_TIMES.getKey(), this.runtimeRetryTimes); runtimeRetryInterval = parseToInt(RUNTIME_RETRY_INTERVAL.getKey(), runtimeRetryInterval); @@ -284,14 +321,31 @@ private void initProperties() { runtimeBatchMaxWait = parseToLong(RUNTIME_BATCH_MAX_WAIT.getKey(), runtimeBatchMaxWait); - rpcConnectTimeout = parseToInt(RPC_CONNECT_TIMEOUT.getKey(), rpcConnectTimeout); - rpcExecuteTimeout = parseToInt(RPC_EXECUTE_TIMEOUT.getKey(), rpcExecuteTimeout); rpcLoginTimeout = parseToInt(RPC_LOGIN_TIMEOUT.getKey(), rpcLoginTimeout); + + slowQueryMonitorThreshold = parseToLong(SLOW_QUERY_MONITOR_THRESHOLD.getKey(), + slowQueryMonitorThreshold); } private void initMetadata() throws Exception { + BOOT.info("begin initMetadata for all tables in database: {}", this.database); + + if (odpMode) { + try { + odpTable = new ObTable.Builder(odpAddr, odpPort) // + .setLoginInfo(tenantName, fullUserName, password, database) // + .setProperties(getProperties()).build(); + } catch (Exception e) { + logger + .warn( + "The addr{}:{} failed to put into table roster, the node status may be wrong, Ignore", + odpAddr, odpPort); + throw e; + } + return; + } this.ocpModel = loadOcpModel(paramURL, dataSourceName, rsListAcquireConnectTimeout, rsListAcquireReadTimeout, rsListAcquireTryTimes, rsListAcquireRetryInterval); @@ -303,19 +357,26 @@ private void initMetadata() throws Exception { OCEANBASE_DATABASE, ALL_DUMMY_TABLE); List rsList = ocpModel.getObServerAddrs(); + BOOT.info("{} success to get rsList, paramURL: {}, rsList: {}，idc2Region: {}", + this.database, paramURL, JSON.toJSON(rsList), JSON.toJSON(ocpModel.getIdc2Region())); TableEntry tableEntry = loadTableEntryRandomly(rsList,// rootServerKey,// tableEntryAcquireConnectTimeout,// - tableEntryAcquireSocketTimeout, sysUA); + tableEntryAcquireSocketTimeout, sysUA, initialized); + BOOT.info("{} success to get tableEntry with rootServerKey all_dummy_tables {}", + this.database, JSON.toJSON(tableEntry)); List replicaLocations = tableEntry.getTableLocation() .getReplicaLocations(); + BOOT.info("{} success to get replicaLocation {}", this.database, + JSON.toJSON(replicaLocations)); + for (ReplicaLocation replicaLocation : replicaLocations) { ObServerInfo info = replicaLocation.getInfo(); ObServerAddr addr = replicaLocation.getAddr(); if (!info.isActive()) { - logger.warn("will not init location {} because status is {}", addr.toString(), + BOOT.warn("will not init location {} because status is {}", addr.toString(), info.getStatus()); continue; } @@ -332,12 +393,20 @@ private void initMetadata() throws Exception { tableRoster.put(addr, obTable); servers.add(addr); } catch (Exception e) { - logger - .warn( - "The addr{}:{} failed to put into table roster, the node status may be wrong, Ignore", - addr.getIp(), addr.getSvrPort()); + BOOT.warn( + "The addr{}:{} failed to put into table roster, the node status may be wrong, Ignore", + addr.getIp(), addr.getSvrPort()); + RUNTIME.warn("initMetadata meet exception", e); + e.printStackTrace(); } } + if (servers.isEmpty()) { + BOOT.error("{} failed to connect any replicaLocation server: {}", this.database, + JSON.toJSON(replicaLocations)); + throw new Exception("failed to connect any replicaLocation server"); + } + + BOOT.info("{} success to build server connection {}", this.database, JSON.toJSON(servers)); this.tableRoster = tableRoster; this.serverRoster.reset(servers); @@ -345,19 +414,38 @@ private void initMetadata() throws Exception { if (StringUtil.isEmpty(currentIDC)) { currentIDC = ZoneUtil.getCurrentIDC(); } + String regionFromOcp = ocpModel.getIdc2Region(currentIDC); + BOOT.info("{} success get currentIDC {}, regionFromOcp {}", this.database, currentIDC, + regionFromOcp); + List ldcServers = getServerLdc(serverRoster, tableEntryAcquireConnectTimeout, tableEntryAcquireSocketTimeout, serverAddressPriorityTimeout, serverAddressCachingTimeout, sysUA); + this.serverRoster.resetServerLdc(ObServerLdcLocation.buildLdcLocation(ldcServers, - currentIDC, ocpModel.getIdc2Region(currentIDC))); + currentIDC, regionFromOcp)); - if (logger.isInfoEnabled()) { - logger.info("finish refresh serverRoster: {}", serverRoster); + if (BOOT.isInfoEnabled()) { + BOOT.info("{} finish refresh serverRoster: {}", this.database, + JSON.toJSON(serverRoster)); + BOOT.info("finish initMetadata for all tables for database {}", this.database); } this.lastRefreshMetadataTimestamp = System.currentTimeMillis(); } + public boolean isOdpMode() { + return odpMode; + } + + public void setOdpMode(boolean odpMode) { + this.odpMode = odpMode; + } + + public ObTable getOdpTable() { + return this.odpTable; + } + private abstract class TableExecuteCallback { private final Object[] rowKey; @@ -365,33 +453,55 @@ private abstract class TableExecuteCallback { this.rowKey = rowKey; } - void checkObTableOperationResult(String ip, int port, ObPayload result) { + void checkObTableOperationResult(String ip, int port, ObPayload request, ObPayload result) { if (result == null) { + RUNTIME.error("client get unexpected NULL result"); throw new ObTableException("client get unexpected NULL result"); } if (!(result instanceof ObTableOperationResult)) { + RUNTIME.error("client get unexpected result: " + result.getClass().getName()); throw new ObTableException("client get unexpected result: " + result.getClass().getName()); } ObTableOperationResult obTableOperationResult = (ObTableOperationResult) result; + ObTableOperationRequest obTableOperationRequest = (ObTableOperationRequest) request; obTableOperationResult.setExecuteHost(ip); obTableOperationResult.setExecutePort(port); - ExceptionUtil - .throwObTableException(ip, port, obTableOperationResult.getSequence(), - obTableOperationResult.getUniqueId(), obTableOperationResult.getHeader() - .getErrno()); + long sequence = obTableOperationResult.getSequence() == 0 ? obTableOperationRequest + .getSequence() : obTableOperationResult.getSequence(); + long uniqueId = obTableOperationResult.getUniqueId() == 0 ? obTableOperationRequest + .getUniqueId() : obTableOperationResult.getUniqueId(); + ExceptionUtil.throwObTableException(ip, port, sequence, uniqueId, + obTableOperationResult.getHeader().getErrno(), obTableOperationResult.getHeader() + .getErrMsg()); + } + + void checkObTableQueryAndMutateResult(String ip, int port, ObPayload result) { + + if (result == null) { + RUNTIME.error("client get unexpected NULL result"); + throw new ObTableException("client get unexpected NULL result"); + } + + if (!(result instanceof ObTableQueryAndMutateResult)) { + RUNTIME.error("client get unexpected result: " + result.getClass().getName()); + throw new ObTableException("client get unexpected result: " + + result.getClass().getName()); + } + // TODO: Add func like throwObTableException() + // which will output the ip / port / error information } - abstract T execute(ObPair obTable) throws Exception; + abstract T execute(ObPair obTable) throws Exception; - /** + /* * Get row key. */ public Object[] getRowKey() { - return rowKey; + return this.rowKey; } } @@ -405,6 +515,9 @@ private T execute(String tableName, TableExecuteCallback callback) throws */ private T execute(String tableName, TableExecuteCallback callback, ObServerRoute route) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } boolean needRefreshTableEntry = false; int tryTimes = 0; long startExecute = System.currentTimeMillis(); @@ -419,40 +532,238 @@ private T execute(String tableName, TableExecuteCallback callback, ObServ + runtimeMaxWait + "/ms"); } tryTimes++; - ObPair obPair = null; + ObPair obPair = null; try { - obPair = getTable(tableName, callback.getRowKey(), needRefreshTableEntry, - tableEntryRefreshIntervalWait, route); + if (odpMode) { + obPair = new ObPair(0L, new ObTableParam(odpTable)); + } else { + obPair = getTable(tableName, callback.getRowKey(), + needRefreshTableEntry, tableEntryRefreshIntervalWait, route); + } T t = callback.execute(obPair); resetExecuteContinuousFailureCount(tableName); return t; - } catch (ObTableReplicaNotReadableException ex) { - if (obPair != null && (tryTimes - 1) < runtimeRetryTimes) { - logger.warn("retry when replica not readable: {}", ex.getMessage()); - route.addToBlackList(obPair.getRight().getIp()); + } catch (Exception ex) { + RUNTIME.error("execute while meet exception", ex); + if (odpMode) { + if ((tryTimes - 1) < runtimeRetryTimes) { + if (ex instanceof ObTableException) { + logger + .warn( + "execute while meet Exception, errorCode: {} , errorMsg: {}, try times {}", + ((ObTableException) ex).getErrorCode(), ex.getMessage(), + tryTimes); + } else { + logger.warn("execute while meet Exception, errorMsg: {}, try times {}", + ex.getMessage(), tryTimes); + } + } else { + throw ex; + } + } else { + if (ex instanceof ObTableReplicaNotReadableException) { + if (obPair != null && (tryTimes - 1) < runtimeRetryTimes) { + logger.warn("retry when replica not readable: {}", ex.getMessage()); + if (!odpMode) { + route.addToBlackList(obPair.getRight().getObTable().getIp()); + } + } else { + logger.warn("exhaust retry when replica not readable: {}", + ex.getMessage()); + RUNTIME.error("replica not readable", ex); + throw ex; + } + } else if (ex instanceof ObTableException + && ((ObTableException) ex).isNeedRefreshTableEntry()) { + needRefreshTableEntry = true; + + logger + .warn( + "refresh table while meet Exception needing refresh, errorCode: {}, errorMsg: {}", + ((ObTableException) ex).getErrorCode(), ex.getMessage()); + if (retryOnChangeMasterTimes && (tryTimes - 1) < runtimeRetryTimes) { + logger + .warn( + "retry while meet Exception needing refresh, errorCode: {} , errorMsg: {},retry times {}", + ((ObTableException) ex).getErrorCode(), ex.getMessage(), + tryTimes); + } else { + calculateContinuousFailure(tableName, ex.getMessage()); + throw ex; + } + } else { + calculateContinuousFailure(tableName, ex.getMessage()); + throw ex; + } + } + } + Thread.sleep(runtimeRetryInterval); + } + } + + private abstract class MutationExecuteCallback { + private final Row rowKey; + private final List keyRanges; + + MutationExecuteCallback(Row rowKey, List keyRanges) { + this.rowKey = rowKey; + this.keyRanges = keyRanges; + } + + void checkResult(String ip, int port, ObPayload request, ObPayload result) { + if (result == null) { + RUNTIME.error("client get unexpected NULL result"); + throw new ObTableException("client get unexpected NULL result"); + } + + if (result instanceof ObTableOperationResult) { + ObTableOperationResult obTableOperationResult = (ObTableOperationResult) result; + ObTableOperationRequest obTableOperationRequest = (ObTableOperationRequest) request; + obTableOperationResult.setExecuteHost(ip); + obTableOperationResult.setExecutePort(port); + long sequence = obTableOperationResult.getSequence() == 0 ? obTableOperationRequest + .getSequence() : obTableOperationResult.getSequence(); + long uniqueId = obTableOperationResult.getUniqueId() == 0 ? obTableOperationRequest + .getUniqueId() : obTableOperationResult.getUniqueId(); + ExceptionUtil.throwObTableException(ip, port, sequence, uniqueId, + obTableOperationResult.getHeader().getErrno(), obTableOperationResult + .getHeader().getErrMsg()); + } else if (result instanceof ObTableQueryAndMutateResult) { + // TODO: Add func like throwObTableException() + // which will output the ip / port / error information + } else { + RUNTIME.error("client get unexpected result: " + result.getClass().getName()); + throw new ObTableException("client get unexpected result: " + + result.getClass().getName()); + } + } + + abstract T execute(ObPair obTable) throws Exception; + + /* + * Get row key. + */ + public Row getRowKey() { + return rowKey; + } + + /* + * Get key ranges. + */ + public List getKeyRanges() { + return keyRanges; + } + + } + + /** + * For mutation + */ + private T executeMutation(String tableName, MutationExecuteCallback callback) + throws Exception { + // force strong read by default, for backward compatibility. + return executeMutation(tableName, callback, getRoute(false)); + } + + /** + * Execute with a route strategy for mutation + */ + private T executeMutation(String tableName, MutationExecuteCallback callback, + ObServerRoute route) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + boolean needRefreshTableEntry = false; + int tryTimes = 0; + long startExecute = System.currentTimeMillis(); + while (true) { + checkStatus(); + long currentExecute = System.currentTimeMillis(); + long costMillis = currentExecute - startExecute; + if (costMillis > runtimeMaxWait) { + throw new ObTableTimeoutExcetion("it has tried " + tryTimes + + " times and it has waited " + costMillis + + "/ms which exceeds response timeout " + + runtimeMaxWait + "/ms"); + } + tryTimes++; + ObPair obPair = null; + try { + if (odpMode) { + obPair = new ObPair(0L, new ObTableParam(odpTable)); } else { - logger.warn("exhaust retry when replica not readable: {}", ex.getMessage()); - throw ex; + if (null != callback.getRowKey()) { + // using row key + obPair = getTable(tableName, callback.getRowKey(), needRefreshTableEntry, + tableEntryRefreshIntervalWait, route); + } else if (null != callback.getKeyRanges()) { + // using scan range + obPair = getTable(tableName, new ObTableQuery(), callback.getKeyRanges(), + needRefreshTableEntry, tableEntryRefreshIntervalWait, route); + } else { + throw new ObTableException("rowkey and scan range are null in mutation"); + } } + T t = callback.execute(obPair); + resetExecuteContinuousFailureCount(tableName); + return t; } catch (Exception ex) { - if (ex instanceof ObTableException - && ((ObTableException) ex).isNeedRefreshTableEntry()) { - needRefreshTableEntry = true; - logger.warn( - "refresh table while meet ObTableMasterChangeException, errorCode: {}", - ((ObTableException) ex).getErrorCode()); - if (retryOnChangeMasterTimes && (tryTimes - 1) < runtimeRetryTimes) { + RUNTIME.error("execute while meet exception", ex); + if (odpMode) { + if ((tryTimes - 1) < runtimeRetryTimes) { + if (ex instanceof ObTableException) { + logger + .warn( + "execute while meet Exception, errorCode: {} , errorMsg: {}, try times {}", + ((ObTableException) ex).getErrorCode(), ex.getMessage(), + tryTimes); + } else { + logger.warn( + "execute while meet Exception, exception: {}, try times {}", ex, + tryTimes); + } + } else { + throw ex; + } + } else { + if (ex instanceof ObTableReplicaNotReadableException) { + if (obPair != null && (tryTimes - 1) < runtimeRetryTimes) { + logger.warn("retry when replica not readable: {}", ex.getMessage()); + if (!odpMode) { + route.addToBlackList(obPair.getRight().getObTable().getIp()); + } + } else { + logger.warn("exhaust retry when replica not readable: {}", + ex.getMessage()); + RUNTIME.error("replica not readable", ex); + throw ex; + } + } else if (ex instanceof ObTableException + && ((ObTableException) ex).isNeedRefreshTableEntry()) { + // if the problem is the lack of row key name, throw directly + if(tableRowKeyElement.get(tableName) == null) { + throw ex; + } + needRefreshTableEntry = true; + logger .warn( - "retry while meet ObTableMasterChangeException, errorCode: {} , retry times {}", - ((ObTableException) ex).getErrorCode(), tryTimes); + "refresh table while meet Exception needing refresh, errorCode: {}, errorMsg: {}", + ((ObTableException) ex).getErrorCode(), ex.getMessage()); + if (retryOnChangeMasterTimes && (tryTimes - 1) < runtimeRetryTimes) { + logger + .warn( + "retry while meet Exception needing refresh, errorCode: {} , errorMsg: {},retry times {}", + ((ObTableException) ex).getErrorCode(), ex.getMessage(), + tryTimes); + } else { + calculateContinuousFailure(tableName, ex.getMessage()); + throw ex; + } } else { calculateContinuousFailure(tableName, ex.getMessage()); throw ex; } - } else { - calculateContinuousFailure(tableName, ex.getMessage()); - throw ex; } } Thread.sleep(runtimeRetryInterval); @@ -461,6 +772,9 @@ private T execute(String tableName, TableExecuteCallback callback, ObServ /** * Calculate continuous failure. + * @param tableName table name + * @param errorMsg err msg + * @throws Exception if failed */ public void calculateContinuousFailure(String tableName, String errorMsg) throws Exception { AtomicLong tempFailures = new AtomicLong(); @@ -469,13 +783,17 @@ public void calculateContinuousFailure(String tableName, String errorMsg) throws if (failures.incrementAndGet() > runtimeContinuousFailureCeiling) { logger.warn("refresh table entry {} while execute failed times exceeded {}, msg: {}", tableName, runtimeContinuousFailureCeiling, errorMsg); - getOrRefreshTableEntry(tableName, true, isTableEntryRefreshIntervalWait()); + getOrRefreshTableEntry(tableName, true, isTableEntryRefreshIntervalWait(), true); failures.set(0); + } else { + logger.warn("error msg: {}, current continues failure count: {}", errorMsg, failures); } } + /** * Reset execute continuous failure count. + * @param tableName table name */ public void resetExecuteContinuousFailureCount(String tableName) { AtomicLong failures = tableContinuousFailures.get(tableName); @@ -488,9 +806,9 @@ public void resetExecuteContinuousFailureCount(String tableName) { * refresh all ob server synchronized just in case rslist has changed, it will not refresh if last refresh time is 1 min ago *

* 1. cannot find table from tables, need refresh tables - * 2. server list refresh failed: {@see com.alipay.oceanbase.obproxy.resource.ObServerStateProcessor#MAX_REFRESH_FAILURE} + * 2. server list refresh failed: {see com.alipay.oceanbase.obproxy.resource.ObServerStateProcessor#MAX_REFRESH_FAILURE} * - * @throws Exception + * @throws Exception if fail */ public void syncRefreshMetadata() throws Exception { @@ -506,9 +824,11 @@ public void syncRefreshMetadata() throws Exception { if (!acquired) { // TODO exception should be classified - throw new ObTableGetException("try to lock metadata refreshing timeout " - + "dataSource:" + dataSourceName + " + refresh timeout:" - + tableEntryRefreshLockTimeout + "."); + String errMsg = "try to lock metadata refreshing timeout " + "dataSource:" + + dataSourceName + " + refresh timeout:" + tableEntryRefreshLockTimeout + + "."; + RUNTIME.error(errMsg); + throw new ObTableGetException(errMsg); } try { @@ -538,7 +858,7 @@ public void syncRefreshMetadata() throws Exception { TableEntry tableEntry = loadTableEntryRandomly(rsList,// allDummyKey,// tableEntryAcquireConnectTimeout,// - tableEntryAcquireSocketTimeout, sysUA); + tableEntryAcquireSocketTimeout, sysUA, initialized); List replicaLocations = tableEntry.getTableLocation() .getReplicaLocations(); @@ -590,12 +910,14 @@ public void syncRefreshMetadata() throws Exception { List ldcServers = getServerLdc(serverRoster, tableEntryAcquireConnectTimeout, tableEntryAcquireSocketTimeout, serverAddressPriorityTimeout, serverAddressCachingTimeout, sysUA); + // reset Server LDC location. + String regionFromOcp = ocpModel.getIdc2Region(currentIDC); this.serverRoster.resetServerLdc(ObServerLdcLocation.buildLdcLocation(ldcServers, - currentIDC, ocpModel.getIdc2Region(currentIDC))); + currentIDC, regionFromOcp)); if (logger.isInfoEnabled()) { - logger.info("finish refresh serverRoster: {}", serverRoster); + logger.info("finish refresh serverRoster: {}", JSON.toJSON(serverRoster)); } this.lastRefreshMetadataTimestamp = System.currentTimeMillis(); } finally { @@ -605,12 +927,131 @@ public void syncRefreshMetadata() throws Exception { } } + /* + * return the table name that need get location + * for global index: return global index table name + * others: return primary table name + * @param dataTableName table name + * @param indexName used index name + * @param scanRangeColumns columns that need to be scaned + * @return the real table name + */ + public String getIndexTableName(final String dataTableName, final String indexName, + List scanRangeColumns, boolean forceRefreshIndexInfo) + throws Exception { + String indexTableName = dataTableName; + if (indexName != null && !indexName.isEmpty() && !indexName.equalsIgnoreCase("PRIMARY")) { + String tmpTableName = constructIndexTableName(dataTableName, indexName); + if (tmpTableName == null) { + throw new ObTableException("index table name is null"); + } + ObIndexInfo indexInfo = getOrRefreshIndexInfo(tmpTableName, forceRefreshIndexInfo); + if (indexInfo == null) { + throw new ObTableException("index info is null, indexTableName:" + tmpTableName); + } + if (indexInfo.getIndexType().isGlobalIndex()) { + indexTableName = tmpTableName; + if (scanRangeColumns.isEmpty()) { + throw new ObTableException( + "query by global index need add all index keys in order, indexTableName:" + + indexTableName); + } else { + addRowKeyElement(indexTableName, + scanRangeColumns.toArray(new String[scanRangeColumns.size()])); + } + } + } + return indexTableName; + } + + public String constructIndexTableName(final String dataTableName, final String indexName) + throws Exception { + // construct index table name + TableEntry entry = tableLocations.get(dataTableName); + Long dataTableId = null; + try { + if (entry == null) { + ObServerAddr addr = serverRoster.getServer(serverAddressPriorityTimeout, + serverAddressCachingTimeout); + dataTableId = getTableIdFromRemote(addr, sysUA, tableEntryAcquireConnectTimeout, + tableEntryAcquireSocketTimeout, tenantName, database, dataTableName); + } else { + dataTableId = entry.getTableId(); + } + } catch (Exception e) { + RUNTIME.error("get index table name exception", e); + throw e; + } + return "__idx_" + dataTableId + "_" + indexName; + } + + public ObIndexInfo getOrRefreshIndexInfo(final String indexTableName, boolean forceRefresh) + throws Exception { + + ObIndexInfo indexInfo = indexinfos.get(indexTableName); + if (!forceRefresh && indexInfo != null) { + return indexInfo; + } + Lock tempLock = new ReentrantLock(); + Lock lock = refreshIndexInfoLocks.putIfAbsent(indexTableName, tempLock); + lock = (lock == null) ? tempLock : lock; + boolean acquired = lock.tryLock(tableEntryRefreshLockTimeout, TimeUnit.MILLISECONDS); + if (!acquired) { + String errMsg = "try to lock index infos refreshing timeout " + "dataSource:" + + dataSourceName + " ,indexTableName:" + indexTableName + " , timeout:" + + tableEntryRefreshLockTimeout + "."; + RUNTIME.error(errMsg); + throw new ObTableEntryRefreshException(errMsg); + } + try { + indexInfo = indexinfos.get(indexTableName); + if (!forceRefresh && indexInfo != null) { + return indexInfo; + } else { + logger.info("index info is not exist, create new index info, indexTableName: {}", + indexTableName); + int serverSize = serverRoster.getMembers().size(); + int refreshTryTimes = tableEntryRefreshTryTimes > serverSize ? serverSize + : tableEntryRefreshTryTimes; + for (int i = 0; i < refreshTryTimes; i++) { + ObServerAddr serverAddr = serverRoster.getServer(serverAddressPriorityTimeout, + serverAddressCachingTimeout); + indexInfo = getIndexInfoFromRemote(serverAddr, sysUA, + tableEntryAcquireConnectTimeout, tableEntryAcquireSocketTimeout, + indexTableName); + if (indexInfo != null) { + indexinfos.put(indexTableName, indexInfo); + } else { + RUNTIME.error("get index info from remote is null, indexTableName: {}", + indexTableName); + } + } + return indexInfo; + } + } catch (Exception e) { + RUNTIME.error("getOrRefresh index info meet exception", e); + throw e; + } finally { + lock.unlock(); + } + } + /** * Get or refresh table entry. + * @param tableName table name + * @param refresh is re-fresh + * @param waitForRefresh wait re-fresh + * @param fetchAll fetch all data from server if needed + * @return this + * @throws Exception if fail */ public TableEntry getOrRefreshTableEntry(final String tableName, final boolean refresh, - final boolean waitForRefresh) throws Exception { + final boolean waitForRefresh, boolean fetchAll) + throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } TableEntry tableEntry = tableLocations.get(tableName); // attempt the cached data and try best to avoid lock if (tableEntry != null) { @@ -621,29 +1062,27 @@ public TableEntry getOrRefreshTableEntry(final String tableName, final boolean r // avoid unnecessary lock long punishInterval = (long) (tableEntryRefreshIntervalBase * Math.pow(2, -serverRoster.getMaxPriority())); - punishInterval = punishInterval <= tableEntryRefreshIntervalCeiling ? punishInterval - : tableEntryRefreshIntervalCeiling; + punishInterval = Math.min(punishInterval, tableEntryRefreshIntervalCeiling); long current = System.currentTimeMillis(); long interval = current - tableEntry.getRefreshTimeMills(); - if (interval < punishInterval) { + long fetchAllInterval = current - tableEntry.getRefreshAllTimeMills(); + if ((fetchAll && (fetchAllInterval < punishInterval)) + || (!fetchAll && (interval < punishInterval))) { if (waitForRefresh) { - if (logger.isInfoEnabled()) { - long toHoldTime = punishInterval - interval; - logger - .info( - "punish table entry {} : table entry refresh time {} punish interval {} current time {}. wait for refresh times {}", - tableName, tableEntry.getRefreshTimeMills(), punishInterval, - current, toHoldTime); - try { - // may have more elegant method ? - Thread.sleep(toHoldTime); - } catch (InterruptedException e) { - logger.error(LCD.convert("01-00018"), tableName, punishInterval, e); - throw new ObTableUnexpectedException("waiting for table entry " - + tableName + " punish interval " - + punishInterval - + " is interrupted."); - } + long toHoldTime = punishInterval - interval; + logger + .info( + "punish table entry {} : table entry refresh time {} punish interval {} current time {}. wait for refresh times {}", + tableName, tableEntry.getRefreshTimeMills(), punishInterval, current, + toHoldTime); + try { + // may have more elegant method ? + Thread.sleep(toHoldTime); + } catch (InterruptedException e) { + RUNTIME.error(LCD.convert("01-00018"), tableName, punishInterval, e); + throw new ObTableUnexpectedException("waiting for table entry " + tableName + + " punish interval " + punishInterval + + " is interrupted."); } } else { return tableEntry; @@ -660,11 +1099,11 @@ public TableEntry getOrRefreshTableEntry(final String tableName, final boolean r boolean acquired = lock.tryLock(tableEntryRefreshLockTimeout, TimeUnit.MILLISECONDS); if (!acquired) { - throw new ObTableEntryRefreshException("try to lock table-entry refreshing timeout " - + "dataSource:" + dataSourceName - + " ,tableName:" + tableName + ", refresh:" - + refresh + " , timeout:" - + tableEntryRefreshLockTimeout + "."); + String errMsg = "try to lock table-entry refreshing timeout " + "dataSource:" + + dataSourceName + " ,tableName:" + tableName + ", refresh:" + refresh + + " , timeout:" + tableEntryRefreshLockTimeout + "."; + RUNTIME.error(errMsg); + throw new ObTableEntryRefreshException(errMsg); } try { @@ -672,40 +1111,67 @@ public TableEntry getOrRefreshTableEntry(final String tableName, final boolean r if (tableEntry != null) { // the server roster is ordered by priority - long interval = (long) (tableEntryRefreshIntervalBase * Math.pow(2, + long punishInterval = (long) (tableEntryRefreshIntervalBase * Math.pow(2, -serverRoster.getMaxPriority())); - interval = interval <= tableEntryRefreshIntervalCeiling ? interval + punishInterval = punishInterval <= tableEntryRefreshIntervalCeiling ? punishInterval : tableEntryRefreshIntervalCeiling; // control refresh frequency less than 100 milli second // just in case of connecting to OB Server failed or change master - if (((System.currentTimeMillis() - tableEntry.getRefreshTimeMills())) < interval) { + long interval = System.currentTimeMillis() - tableEntry.getRefreshTimeMills(); + long fetchAllInterval = System.currentTimeMillis() + - tableEntry.getRefreshAllTimeMills(); + if ((fetchAll && (fetchAllInterval < punishInterval)) + || (!fetchAll && (interval < punishInterval))) { return tableEntry; } } if (tableEntry == null || refresh) {// not exist or need refresh, create new table entry + if (logger.isInfoEnabled()) { + if (tableEntry == null) { + logger.info("tableEntry not exist, create new table entry, tablename: {}", + tableName); + } else { + logger.info( + "tableEntry need refresh, create new table entry, tablename: {}", + tableName); + } + } + int serverSize = serverRoster.getMembers().size(); int refreshTryTimes = tableEntryRefreshTryTimes > serverSize ? serverSize : tableEntryRefreshTryTimes; for (int i = 0; i < refreshTryTimes; i++) { try { - return refreshTableEntry(tableEntry, tableName); + return refreshTableEntry(tableEntry, tableName, fetchAll); } catch (ObTableNotExistException e) { + RUNTIME.error("getOrRefreshTableEntry meet exception", e); throw e; } catch (ObTableServerCacheExpiredException e) { + RUNTIME.error("getOrRefreshTableEntry meet exception", e); + if (logger.isInfoEnabled()) { logger.info("server addr is expired and it will refresh metadata."); } syncRefreshMetadata(); tableEntryRefreshContinuousFailureCount.set(0); } catch (ObTableEntryRefreshException e) { + RUNTIME.error("getOrRefreshTableEntry meet exception", e); + // if the problem is the lack of row key name, throw directly + if (tableRowKeyElement.get(tableName) == null) { + throw e; + } + if (tableEntryRefreshContinuousFailureCount.incrementAndGet() > tableEntryRefreshContinuousFailureCeiling) { logger.error(LCD.convert("01-00019"), tableEntryRefreshContinuousFailureCeiling); syncRefreshMetadata(); tableEntryRefreshContinuousFailureCount.set(0); } + } catch (Throwable t) { + RUNTIME.error("getOrRefreshTableEntry meet exception", t); + throw t; } } // failure reach the try times may all the server change @@ -724,14 +1190,33 @@ public TableEntry getOrRefreshTableEntry(final String tableName, final boolean r } } + /** + * 刷新 table entry 元数据 + * @param tableEntry + * @param tableName + * @return + * @throws ObTableEntryRefreshException + */ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName) throws ObTableEntryRefreshException { + return refreshTableEntry(tableEntry, tableName, false); + } + + /** + * 刷新 table entry 元数据 + * @param tableEntry + * @param tableName + * @param fetchAll + * @return + * @throws ObTableEntryRefreshException + */ + private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName, boolean fetchAll) + throws ObTableEntryRefreshException { TableEntryKey tableEntryKey = new TableEntryKey(clusterName, tenantName, database, tableName); try { - // if table entry is exist we just need to refresh table locations - if (tableEntry != null) { + if (tableEntry != null && !fetchAll) { tableEntry = loadTableEntryLocationWithPriority(serverRoster, // tableEntryKey,// tableEntry,// @@ -751,6 +1236,7 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName) if (tableEntry.isPartitionTable()) { switch (runningMode) { case HBASE: + tableRowKeyElement.put(tableName, HBASE_ROW_KEY_ELEMENT); tableEntry.setRowKeyElement(HBASE_ROW_KEY_ELEMENT); break; case NORMAL: @@ -758,9 +1244,11 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName) if (rowKeyElement != null) { tableEntry.setRowKeyElement(rowKeyElement); } else { + RUNTIME.error("partition table must add row key element name for table: " + tableName + " with table entry key: " + + tableEntryKey); throw new ObTableEntryRefreshException( - "partition table must has row key element key =" - + tableEntryKey); + "partition table must add row key element name for table: " + tableName + " with table entry key: " + + tableEntryKey); } } tableEntry.prepare(); @@ -769,81 +1257,350 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName) // prepare the table entry for weak read. tableEntry.prepareForWeakRead(serverRoster.getServerLdcLocation()); } catch (ObTableNotExistException e) { + RUNTIME.error("refreshTableEntry meet exception", e); throw e; } catch (ObTableServerCacheExpiredException e) { + RUNTIME.error("refreshTableEntry meet exception", e); throw e; } catch (Exception e) { - logger.error(LCD.convert("01-00020"), tableEntryKey, tableEntry, e); + RUNTIME.error(LCD.convert("01-00020"), tableEntryKey, tableEntry, e); throw new ObTableEntryRefreshException(String.format( "failed to get table entry key=%s original tableEntry=%s ", tableEntryKey, tableEntry), e); } tableLocations.put(tableName, tableEntry); + if (fetchAll) { + tableEntry.setRefreshAllTimeMills(System.currentTimeMillis()); + } tableEntryRefreshContinuousFailureCount.set(0); if (logger.isInfoEnabled()) { logger.info( "refresh table entry, dataSource: {}, tableName: {}, refresh: {} key:{} entry:{} ", - dataSourceName, tableName, true, tableEntryKey, tableEntry); + dataSourceName, tableName, true, tableEntryKey, JSON.toJSON(tableEntry)); } return tableEntry; } - private static final Long MASK = (1L << PART_ID_BITNUM) - | 1L << (PART_ID_BITNUM + PART_ID_SHIFT); - - private long getPartition(TableEntry tableEntry, Object[] rowKey) { - // non partition - if (!tableEntry.isPartitionTable() - || tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ZERO) { - return 0L; - } - + /** + * 根据 tableGroup 获取其中一个tableName + * physicalTableName Complete table from table group + * @param physicalTableName + * @param tableGroupName + * @return + * @throws Exception + */ + private String refreshTableNameByTableGroup(String physicalTableName, String tableGroupName) + throws Exception { + TableEntryKey tableEntryKey = new TableEntryKey(clusterName, tenantName, database, + tableGroupName); + String oldTableName = physicalTableName; + try { + physicalTableName = loadTableNameWithGroupName(serverRoster, // + tableEntryKey,// + tableEntryAcquireConnectTimeout,// + tableEntryAcquireSocketTimeout,// + serverAddressPriorityTimeout,// + serverAddressCachingTimeout, sysUA); + } catch (ObTableNotExistException e) { + RUNTIME.error("refreshTableNameByTableGroup from tableGroup meet exception", e); + throw e; + } catch (ObTableServerCacheExpiredException e) { + RUNTIME.error("refreshTableEntry from tableGroup meet exception", e); + throw e; + } catch (Exception e) { + RUNTIME.error("refreshTableEntry from tableGroup meet exception", tableEntryKey, + physicalTableName, e); + throw new ObTableNotExistException(String.format( + "failed to get table name key=%s original tableName=%s ", tableEntryKey, + physicalTableName), e); + } + if (!TableGroupInverted.isEmpty() && oldTableName != null + && TableGroupInverted.containsKey(oldTableName)) { + TableGroupInverted.remove(oldTableName, tableGroupName); + } + TableGroupCache.put(tableGroupName, physicalTableName); + TableGroupInverted.put(physicalTableName, tableGroupName); + if (logger.isInfoEnabled()) { + logger + .info( + "get table name from tableGroup, dataSource: {}, tableName: {}, refresh: {} key:{} realTableName:{} ", + dataSourceName, tableGroupName, true, tableEntryKey, physicalTableName); + } + return physicalTableName; + } + + /** + * 根据 rowkey 获取分区 id + * @param tableEntry + * @param row + * @return + */ + private long getPartition(TableEntry tableEntry, Row row) { + // non partition + if (!tableEntry.isPartitionTable() + || tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ZERO) { + return 0L; + } if (tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ONE) { - return tableEntry.getPartitionInfo().getFirstPartDesc().getPartId(rowKey); + return tableEntry.getPartitionInfo().getFirstPartDesc().getPartId(row); + } + + Long partId1 = tableEntry.getPartitionInfo().getFirstPartDesc().getPartId(row); + Long partId2 = tableEntry.getPartitionInfo().getSubPartDesc().getPartId(row); + return generatePartId(partId1, partId2); + } + + /* + * Get logicId(partition id in 3.x) from giving range + */ + private List getPartitionsForLevelTwo(TableEntry tableEntry, Row startRow, + boolean startIncluded, Row endRow, + boolean endIncluded) throws Exception { + if (tableEntry.getPartitionInfo().getLevel() != ObPartitionLevel.LEVEL_TWO) { + RUNTIME.error("getPartitionsForLevelTwo need ObPartitionLevel LEVEL_TWO"); + throw new Exception("getPartitionsForLevelTwo need ObPartitionLevel LEVEL_TWO"); + } + + List partIds1 = tableEntry.getPartitionInfo().getFirstPartDesc() + .getPartIds(startRow, startIncluded, endRow, endIncluded); + List partIds2 = tableEntry.getPartitionInfo().getSubPartDesc() + .getPartIds(startRow, startIncluded, endRow, endIncluded); + + List partIds = new ArrayList(); + if (partIds1.isEmpty()) { + // do nothing + } else if (partIds1.size() == 1) { + long firstPartId = partIds1.get(0); + for (Long partId2 : partIds2) { + partIds.add(generatePartId(firstPartId, partId2)); + } + } else { + // construct all sub partition idx + long subPartNum = tableEntry.getPartitionInfo().getSubPartDesc().getPartNum(); + List subPartIds = new ArrayList(); + for (long i = 0; i < subPartNum; i++) { + subPartIds.add(i); + } + partIds2 = Collections.unmodifiableList(subPartIds); + + for (Long partId1 : partIds1) { + for (Long partId2 : partIds2) { + partIds.add(generatePartId(partId1, partId2)); + } + } } - Long partId1 = tableEntry.getPartitionInfo().getFirstPartDesc().getPartId(rowKey); - Long partId2 = tableEntry.getPartitionInfo().getSubPartDesc().getPartId(rowKey); - return (partId1 << PART_ID_SHIFT) | partId2 | MASK; + return partIds; } + /** + * + * @param tableEntry + * @param partId accept logic id (partId partitionId in 3.x) + * @param route + * @return + */ private ObPair getPartitionReplica(TableEntry tableEntry, long partId, ObServerRoute route) { return new ObPair(partId, getPartitionLocation(tableEntry, partId, route)); } + /** + * + * @param tableEntry + * @param partId accept logic id (partId partitionId in 3.x) + * @param route + * @return + */ private ReplicaLocation getPartitionLocation(TableEntry tableEntry, long partId, ObServerRoute route) { - return tableEntry.getPartitionEntry().getPartitionLocationWithPartId(partId) - .getReplica(route); + if (ObGlobal.obVsnMajor() >= 4 && tableEntry.isPartitionTable()) { + ObPartitionInfo partInfo = tableEntry.getPartitionInfo(); + Map tabletIdMap = partInfo.getPartTabletIdMap(); + long partIdx = tableEntry.getPartIdx(partId); + long TabletId = tabletIdMap.get(partIdx); + return tableEntry.getPartitionEntry().getPartitionLocationWithTabletId(TabletId) + .getReplica(route); + } else { + return tableEntry.getPartitionEntry().getPartitionLocationWithPartId(partId) + .getReplica(route); + } } - public ObPair getTable(String tableName, Object[] rowKey, boolean refresh, - boolean waitForRefresh) throws Exception { - return getTable(tableName, rowKey, refresh, waitForRefresh, getRoute(false)); + /** + * + * @param tableName table want to get + * @param rowKey row key + * @param refresh whether to refresh + * @param waitForRefresh whether wait for refresh + * @return ObPair of partId and table + * @throws Exception exception + */ + public ObPair getTable(String tableName, Object[] rowKey, + boolean refresh, boolean waitForRefresh) + throws Exception { + ObServerRoute route = getRoute(false); + return getTable(tableName, rowKey, refresh, waitForRefresh, route); } - public ObPair getTable(String tableName, Object[] rowKey, boolean refresh, - boolean waitForRefresh, ObServerRoute route) - throws Exception { - TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh); + /** + * + * @param tableName table want to get + * @param rowKey row key + * @param refresh whether to refresh + * @param waitForRefresh whether wait for refresh + * @param route ObServer route + * @return ObPair of partId and table + * @throws Exception exception + */ + public ObPair getTable(String tableName, Object[] rowKey, boolean refresh, + boolean waitForRefresh, ObServerRoute route) + throws Exception { + TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh, false); + Row row = new Row(); + if (tableEntry.isPartitionTable() + && tableEntry.getPartitionInfo().getLevel() != ObPartitionLevel.LEVEL_ZERO) { + List curTableRowKeyNames = new ArrayList(); + Map tableRowKeyEle = getRowKeyElement(tableName); + if (tableRowKeyEle != null) { + curTableRowKeyNames = new ArrayList(tableRowKeyEle.keySet()); + } + if (curTableRowKeyNames.isEmpty()) { + throw new IllegalArgumentException("Please make sure add row key elements"); + } + + // match the correct key to its row key + for (int i = 0; i < rowKey.length; ++i) { + if (i < curTableRowKeyNames.size()) { + row.add(curTableRowKeyNames.get(i), rowKey[i]); + } + else { // the rowKey element in the table only contain partition key(s) or the input row key has redundant elements + break; + } + } + } - long partId = getPartition(tableEntry, rowKey); + long partId = getPartition(tableEntry, row); // partition id in 3.x, origin partId in 4.x, logicId - return getTable(tableName, tableEntry, partId, waitForRefresh, route); + return getTableInternal(tableName, tableEntry, partId, waitForRefresh, route); } - public ObPair getTable(String tableName, long partId, boolean refresh, + /** + * For mutation (queryWithFilter) + * @param tableName table want to get + * @param keyRanges key + * @param refresh whether to refresh + * @param waitForRefresh whether wait for refresh + * @param route ObServer route + * @return ObPair of partId and table + * @throws Exception exception + */ + public ObPair getTable(String tableName, ObTableQuery query, List keyRanges, boolean refresh, boolean waitForRefresh, ObServerRoute route) throws Exception { - return getTable(tableName, getOrRefreshTableEntry(tableName, refresh, waitForRefresh), - partId, waitForRefresh, route); + Map partIdMapObTable = new HashMap(); + for (ObNewRange rang : keyRanges) { + ObRowKey startKey = rang.getStartKey(); + int startKeySize = startKey.getObjs().size(); + ObRowKey endKey = rang.getEndKey(); + int endKeySize = endKey.getObjs().size(); + Object[] start = new Object[startKeySize]; + Object[] end = new Object[endKeySize]; + for (int i = 0; i < startKeySize; i++) { + start[i] = startKey.getObj(i).getValue(); + } + + for (int i = 0; i < endKeySize; i++) { + end[i] = endKey.getObj(i).getValue(); + } + ObBorderFlag borderFlag = rang.getBorderFlag(); + List> pairList = getTables(tableName, query, start, + borderFlag.isInclusiveStart(), end, borderFlag.isInclusiveEnd(), false, + false); + for (ObPair pair : pairList) { + partIdMapObTable.put(pair.getLeft(), pair.getRight()); + } + } + + if (partIdMapObTable.size() > 1) { + throw new ObTablePartitionConsistentException( + "query and mutate must be a atomic operation"); + } else if (partIdMapObTable.size() < 1) { + throw new ObTableException("could not find part id of range"); + } + + ObPair ans = null; + for (Long partId: partIdMapObTable.keySet()) { + ans = new ObPair<>(partId, partIdMapObTable.get(partId)); + } + return ans; } - public ObPair getTable(String tableName, TableEntry tableEntry, long partId, - boolean waitForRefresh, ObServerRoute route) - throws Exception { + /** + * For mutation execute + * @param tableName table want to get + * @param rowKey row key with column names + * @param refresh whether to refresh + * @param waitForRefresh whether wait for refresh + * @return ObPair of partId and table + * @throws Exception exception + */ + public ObPair getTable(String tableName, Row rowKey, boolean refresh, + boolean waitForRefresh) throws Exception { + return getTable(tableName, rowKey, refresh, waitForRefresh, getRoute(false)); + } + + /** + * For mutation execute + * @param tableName table want to get + * @param rowKey row key with column names + * @param refresh whether to refresh + * @param waitForRefresh whether wait for refresh + * @param route ObServer route + * @return ObPair of partId and table + * @throws Exception exception + */ + public ObPair getTable(String tableName, Row rowKey, boolean refresh, + boolean waitForRefresh, ObServerRoute route) + throws Exception { + TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh, false); + long partId; + partId = getPartition(tableEntry, rowKey); // partition id in 3.x, origin partId in 4.x, logicId + + return getTableInternal(tableName, tableEntry, partId, waitForRefresh, route); + } + + /** + * get addr by pardId + * @param tableName table want to get + * @param partId partId of table (logicId, partition id in 3.x) + * @param refresh whether to refresh + * @param waitForRefresh whether wait for refresh + * @param route ObServer route + * @return ObPair of partId and table + * @throws Exception exception + */ + public ObPair getTableWithPartId(String tableName, long partId, + boolean refresh, boolean waitForRefresh, boolean needFetchAll, + ObServerRoute route) throws Exception { + TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh, needFetchAll); + return getTableInternal(tableName, tableEntry, partId, waitForRefresh, route); + } + + /** + * get addr from table entry by pardId + * @param tableName table want to get + * @param tableEntry tableEntry + * @param partId partId of table (logicId, partition id in 3.x) + * @param waitForRefresh whether wait for refresh + * @param route ObServer route + * @return ObPair of partId and table + * @throws Exception exception + */ + public ObPair getTableInternal(String tableName, TableEntry tableEntry, + long partId, boolean waitForRefresh, + ObServerRoute route) throws Exception { ObPair partitionReplica = getPartitionReplica(tableEntry, partId, route); @@ -860,25 +1617,52 @@ public ObPair getTable(String tableName, TableEntry tableEntry, l if (logger.isInfoEnabled() && addrExpired) { logger.info("server addr {} is expired, refresh tableEntry.", addr); } - tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh); + + tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh, false); replica = getPartitionReplica(tableEntry, partId, route).getRight(); addr = replica.getAddr(); obTable = tableRoster.get(addr); } if (obTable == null) { + RUNTIME.error("cannot get table by addr: " + addr); throw new ObTableGetException("cannot get table by addr: " + addr); } + + ObTableParam param = new ObTableParam(obTable); + param.setPartId(partId); // used in getTable(), 4.x may change the origin partId + if (ObGlobal.obVsnMajor() >= 4 && tableEntry != null) { + long partIdx = tableEntry.getPartIdx(partId); + partId = tableEntry.isPartitionTable() ? tableEntry.getPartitionInfo() + .getPartTabletIdMap().get(partIdx) : partId; + param.setLsId(tableEntry.getPartitionEntry().getLsId(partId)); + } + + param.setTableId(tableEntry.getTableId()); + param.setPartitionId(partId); + addr.recordAccess(); - return new ObPair(partitionReplica.getLeft(), obTable); + return new ObPair(partitionReplica.getLeft(), param); } + /** + * 根据 start-end 获取 partition id 和 addr + * @param tableEntry + * @param startRow + * @param startIncluded + * @param endRow + * @param endIncluded + * @param route + * @return + * @throws Exception + */ private List> getPartitionReplica(TableEntry tableEntry, - Object[] start, + Row startRow, boolean startIncluded, - Object[] end, + Row endRow, boolean endIncluded, - ObServerRoute route) { + ObServerRoute route) + throws Exception { // non partition List> replicas = new ArrayList>(); if (!tableEntry.isPartitionTable() @@ -888,39 +1672,102 @@ private List> getPartitionReplica(TableEntry table return replicas; } else if (tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ONE) { List partIds = tableEntry.getPartitionInfo().getFirstPartDesc() - .getPartIds(start, startIncluded, end, endIncluded); + .getPartIds(startRow, startIncluded, endRow, endIncluded); + for (Long partId : partIds) { + replicas.add(new ObPair(partId, getPartitionLocation( + tableEntry, partId, route))); + } + } else if (tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_TWO) { + List partIds = getPartitionsForLevelTwo(tableEntry, startRow, startIncluded, + endRow, endIncluded); for (Long partId : partIds) { replicas.add(new ObPair(partId, getPartitionLocation( tableEntry, partId, route))); } } else { - throw new ObTableGetException("range partition is not allowed in level two"); + RUNTIME.error("not allowed bigger than level two"); + throw new ObTableGetException("not allowed bigger than level two"); } return replicas; } - public List> getTables(String tableName, Object[] start, - boolean startInclusive, Object[] end, - boolean endInclusive, boolean refresh, - boolean waitForRefresh) throws Exception { - return getTables(tableName, start, startInclusive, end, endInclusive, refresh, + /** + * 根据 start-end 获取 partition ids 和 addrs + * @param tableName table want to get + * @param start start key + * @param startInclusive whether include start key + * @param end end key + * @param endInclusive whether include end key + * @param refresh whether to refresh + * @param waitForRefresh whether wait for refresh + * @return list of ObPair of partId(logicId) and table obTableParams + * @throws Exception exception + */ + public List> getTables(String tableName, ObTableQuery query, + Object[] start, boolean startInclusive, + Object[] end, boolean endInclusive, + boolean refresh, boolean waitForRefresh) + throws Exception { + return getTables(tableName, query, start, startInclusive, end, endInclusive, refresh, waitForRefresh, getRoute(false)); } - public List> getTables(String tableName, Object[] start, - boolean startInclusive, Object[] end, - boolean endInclusive, boolean refresh, - boolean waitForRefresh, ObServerRoute route) - throws Exception { + /** + * 根据 start-end 获取 partition id 和 addr + * @param tableName table want to get + * @param start start key + * @param startInclusive whether include start key + * @param end end key + * @param endInclusive whether include end key + * @param refresh whether to refresh + * @param waitForRefresh whether wait for refresh + * @param route server route + * @return list of ObPair of partId(logicId) and tableParam + * @throws Exception exception + */ + public List> getTables(String tableName, ObTableQuery query, Object[] start, + boolean startInclusive, Object[] end, + boolean endInclusive, boolean refresh, + boolean waitForRefresh, ObServerRoute route) + throws Exception { // 1. get TableEntry information - TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh); + TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh, false); + + List scanRangeColumns = query.getScanRangeColumns(); + if (scanRangeColumns == null || scanRangeColumns.isEmpty()) { + Map tableEntryRowKeyElement = getRowKeyElement(tableName); + if (tableEntryRowKeyElement != null) { + scanRangeColumns = new ArrayList(tableEntryRowKeyElement.keySet()); + } + } // 2. get replica location + // partIdWithReplicaList -> List> + if (start.length != end.length) { + throw new IllegalArgumentException("length of start key and end key is not equal"); + } + + Row startRow = new Row(); + Row endRow = new Row(); + // ensure the format of column names and values if the current table is a table with partition + if (tableEntry.isPartitionTable() && tableEntry.getPartitionInfo().getLevel() != ObPartitionLevel.LEVEL_ZERO) { + // scanRangeColumn may be longer than start/end in prefix scanning situation + if (scanRangeColumns == null || scanRangeColumns.size() < start.length) { + throw new IllegalArgumentException( + "length of key and scan range columns do not match, please use addRowKeyElement or set scan range columns"); + } + for (int i = 0; i < start.length; i++) { + startRow.add(scanRangeColumns.get(i), start[i]); + endRow.add(scanRangeColumns.get(i), end[i]); + } + } + List> partIdWithReplicaList = getPartitionReplica(tableEntry, - start, startInclusive, end, endInclusive, route); + startRow, startInclusive, endRow, endInclusive, route); - List> obTables = new ArrayList>(); + // obTableParams -> List> + List> obTableParams = new ArrayList>(); for (ObPair partIdWithReplica : partIdWithReplicaList) { long partId = partIdWithReplica.getLeft(); ReplicaLocation replica = partIdWithReplica.getRight(); @@ -933,20 +1780,119 @@ public List> getTables(String tableName, Object[] start, "server address {} is expired={} or can not get ob table. So that will sync refresh metadata", addr, addrExpired); syncRefreshMetadata(); - tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh); + tableEntry = getOrRefreshTableEntry(tableName, true, waitForRefresh, false); replica = getPartitionLocation(tableEntry, partId, route); addr = replica.getAddr(); obTable = tableRoster.get(addr); } if (obTable == null) { + RUNTIME.error("cannot get table by addr: " + addr); throw new ObTableGetException("cannot get table by addr: " + addr); } + + ObTableParam param = new ObTableParam(obTable); + if (ObGlobal.obVsnMajor() >= 4) { + long partIdx = tableEntry.getPartIdx(partId); + partId = tableEntry.isPartitionTable() ? tableEntry.getPartitionInfo() + .getPartTabletIdMap().get(partIdx) : partId; + } + + param.setTableId(tableEntry.getTableId()); + // real partition(tablet) id + param.setPartitionId(partId); + addr.recordAccess(); + obTableParams.add(new ObPair(partIdWithReplica.getLeft(), param)); + } + + return obTableParams; + } + + /** + * get table name with table group + * @param tableGroupName table group name + * @param refresh if refresh or not + * @return actual table name + * @throws Exception exception + */ + public String tryGetTableNameFromTableGroupCache(final String tableGroupName, + final boolean refresh) throws Exception { + String physicalTableName = TableGroupCache.get(tableGroupName); // tableGroup -> Table + // get tableName from cache + if (physicalTableName != null && !refresh) { + return physicalTableName; + } + + // not find in cache, should get tableName from observer + Lock tempLock = new ReentrantLock(); + Lock lock = TableGroupCacheLocks.putIfAbsent(tableGroupName, tempLock); + lock = (lock == null) ? tempLock : lock; // check the first lock + + // attempt lock the refreshing action, avoiding concurrent refreshing + // use the time-out mechanism, avoiding the rpc hanging up + boolean acquired = lock.tryLock(metadataRefreshLockTimeout, TimeUnit.MILLISECONDS); - obTables.add(new ObPair(partId, obTable)); + if (!acquired) { + String errMsg = "try to lock tableGroup inflect timeout " + "dataSource:" + + dataSourceName + " ,tableName:" + tableGroupName + " , timeout:" + + metadataRefreshLockTimeout + "."; + RUNTIME.error(errMsg); + throw new ObTableEntryRefreshException(errMsg); + } + + try { + String newPhyTableName = TableGroupCache.get(tableGroupName); + if (((physicalTableName == null) && (newPhyTableName == null)) + || (refresh && newPhyTableName.equalsIgnoreCase(physicalTableName))) { + if (logger.isInfoEnabled()) { + if (physicalTableName != null) { + logger.info( + "realTableName need refresh, create new table entry, tablename: {}", + tableGroupName); + } else { + logger.info( + "realTableName not exist, create new table entry, tablename: {}", + tableGroupName); + } + } + + try { + return refreshTableNameByTableGroup(physicalTableName, tableGroupName); + } catch (ObTableNotExistException e) { + RUNTIME.error("getOrRefreshTableName from TableGroup meet exception", e); + throw e; + } catch (ObTableServerCacheExpiredException e) { + RUNTIME.error("getOrRefreshTableName from TableGroup meet exception", e); + + if (logger.isInfoEnabled()) { + logger.info("server addr is expired and it will refresh metadata."); + } + syncRefreshMetadata(); + } catch (Throwable t) { + RUNTIME.error("getOrRefreshTableName from TableGroup meet exception", t); + throw t; + } + // failure reach the try times may all the server change + if (logger.isInfoEnabled()) { + logger.info("refresh table Name from TableGroup failure"); + } + } + return newPhyTableName; + } finally { + lock.unlock(); } - return obTables; + } + + /** + * Aggregate. + * @param tableName table want to aggregate + * @return ObTableAggregation object + */ + public ObTableAggregation aggregate(String tableName) { + ObTableClientQueryImpl tableQuery = new ObTableClientQueryImpl(tableName, this); + ObClusterTableQuery clusterTableQuery = new ObClusterTableQuery(tableQuery); + return new ObTableAggregation(clusterTableQuery); } /** @@ -959,11 +1905,6 @@ public TableQuery query(String tableName) { return new ObClusterTableQuery(tableQuery); } - @Override - public TableQuery queryByBatch(String tableName) throws Exception { - return new QueryByBatch(query(tableName)); - } - /** * Batch. */ @@ -977,187 +1918,881 @@ public TableBatchOps batch(String tableName) { @Override public Map get(final String tableName, final Object[] rowKey, final String[] columns) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + final long startTime = System.currentTimeMillis(); final ObReadConsistency obReadConsistency = this.getReadConsistency(); return execute(tableName, new TableExecuteCallback>(rowKey) { @Override - public Map execute(ObPair obPair) throws Exception { - ObTable obTable = obPair.getRight(); - long partId = obPair.getLeft(); + public Map execute(ObPair obPair) throws Exception { + long getTableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); ObTableOperationRequest request = ObTableOperationRequest.getInstance(tableName, GET, rowKey, columns, null, obTable.getObTableOperationTimeout()); - request.setPartitionId(partId); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); request.setConsistencyLevel(obReadConsistency.toObTableConsistencyLevel()); - ObPayload result = obPair.getRight().execute(request); - checkObTableOperationResult(obTable.getIp(), obTable.getPort(), result); + ObPayload result = obTable.execute(request); + checkObTableOperationResult(obTable.getIp(), obTable.getPort(), request, result); + + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "GET", endpoint, rowKey, + (ObTableOperationResult) result, getTableTime - startTime, + System.currentTimeMillis() - getTableTime, getslowQueryMonitorThreshold()); return ((ObTableOperationResult) result).getEntity().getSimpleProperties(); } }, getReadRoute()); } + /** + * Update. + */ + public Update update(String tableName) { + return new Update(this, tableName); + } + /** * Update. */ @Override public long update(final String tableName, final Object[] rowKey, final String[] columns, final Object[] values) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + final long start = System.currentTimeMillis(); return execute(tableName, new TableExecuteCallback(rowKey) { /** * Execute. */ @Override - public Long execute(ObPair obPair) throws Exception { - ObTable obTable = obPair.getRight(); - long partId = obPair.getLeft(); + public Long execute(ObPair obPair) throws Exception { + long getTableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); ObTableOperationRequest request = ObTableOperationRequest.getInstance(tableName, UPDATE, rowKey, columns, values, obTable.getObTableOperationTimeout()); - request.setPartitionId(partId); - ObPayload result = obPair.getRight().execute(request); - checkObTableOperationResult(obTable.getIp(), obTable.getPort(), result); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "UPDATE", endpoint, rowKey, + (ObTableOperationResult) result, getTableTime - start, + System.currentTimeMillis() - getTableTime, getslowQueryMonitorThreshold()); + checkObTableOperationResult(obTable.getIp(), obTable.getPort(), request, result); return ((ObTableOperationResult) result).getAffectedRows(); } }); } + /** + * Update with result + * @param tableName which table to update + * @param rowKey update row key + * @param keyRanges scan range + * @param columns columns name to update + * @param values new values + * @return execute result + * @throws Exception exception + */ + public ObPayload updateWithResult(final String tableName, final Row rowKey, + final List keyRanges, final String[] columns, + final Object[] values) throws Exception { + final long start = System.currentTimeMillis(); + return executeMutation(tableName, + new MutationExecuteCallback(rowKey, keyRanges) { + /** + * Execute. + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, UPDATE, rowKey.getValues(), columns, values, + obTable.getObTableOperationTimeout()); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "UPDATE", endpoint, rowKey.getValues(), + (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + + /** + * Delete. + */ + public Delete delete(String tableName) { + return new Delete(this, tableName); + } + /** * Delete. */ @Override public long delete(final String tableName, final Object[] rowKey) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + final long start = System.currentTimeMillis(); return execute(tableName, new TableExecuteCallback(rowKey) { /** * Execute. */ @Override - public Long execute(ObPair obPair) throws Exception { - ObTable obTable = obPair.getRight(); - long partId = obPair.getLeft(); + public Long execute(ObPair obPair) throws Exception { + long getTableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); ObTableOperationRequest request = ObTableOperationRequest.getInstance(tableName, DEL, rowKey, null, null, obTable.getObTableOperationTimeout()); - request.setPartitionId(partId); - ObPayload result = obPair.getRight().execute(request); - checkObTableOperationResult(obTable.getIp(), obTable.getPort(), result); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "DELETE", endpoint, rowKey, + (ObTableOperationResult) result, getTableTime - start, + System.currentTimeMillis() - getTableTime, getslowQueryMonitorThreshold()); + checkObTableOperationResult(obTable.getIp(), obTable.getPort(), request, result); return ((ObTableOperationResult) result).getAffectedRows(); } }); } + /** + * Delete with result + * @param tableName which table to delete + * @param rowKey delete row key + * @param keyRanges scan range + * @return execute result + * @throws Exception exception + */ + public ObPayload deleteWithResult(final String tableName, final Row rowKey, + final List keyRanges) throws Exception { + final long start = System.currentTimeMillis(); + return executeMutation(tableName, + new MutationExecuteCallback(rowKey, keyRanges) { + + /** + * Execute. + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, DEL, rowKey.getValues(), null, null, obTable.getObTableOperationTimeout()); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "DELETE", endpoint, rowKey.getValues(), + (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + + /** + * Insert. + */ + public Insert insert(String tableName) { + return new Insert(this, tableName); + } + /** * Insert. */ @Override public long insert(final String tableName, final Object[] rowKey, final String[] columns, final Object[] values) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + final long start = System.currentTimeMillis(); return execute(tableName, new TableExecuteCallback(rowKey) { /** * Execute. */ @Override - public Long execute(ObPair obPair) throws Exception { - ObTable obTable = obPair.getRight(); - long partId = obPair.getLeft(); + public Long execute(ObPair obPair) throws Exception { + long getTableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); ObTableOperationRequest request = ObTableOperationRequest.getInstance(tableName, INSERT, rowKey, columns, values, obTable.getObTableOperationTimeout()); - request.setPartitionId(partId); - ObPayload result = obPair.getRight().execute(request); - checkObTableOperationResult(obTable.getIp(), obTable.getPort(), result); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "INSERT", endpoint, rowKey, + (ObTableOperationResult) result, getTableTime - start, + System.currentTimeMillis() - getTableTime, getslowQueryMonitorThreshold()); + checkObTableOperationResult(obTable.getIp(), obTable.getPort(), request, result); return ((ObTableOperationResult) result).getAffectedRows(); } }); } + /** + * Insert with result + * @param tableName which table to insert + * @param rowKey insert row key + * @param keyRanges scan range + * @param columns columns name to insert + * @param values new values + * @return execute result + * @throws Exception exception + */ + public ObPayload insertWithResult(final String tableName, final Row rowKey, + final List keyRanges, final String[] columns, + final Object[] values) throws Exception { + final long start = System.currentTimeMillis(); + return executeMutation(tableName, + new MutationExecuteCallback(rowKey, keyRanges) { + /** + * Execute. + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, INSERT, rowKey.getValues(), columns, values, + obTable.getObTableOperationTimeout()); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "INSERT", endpoint, rowKey.getValues(), + (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + + /** + * put with result + * @param tableName which table to put + * @param rowKey insert row key + * @param keyRanges scan range + * @param columns columns name to put + * @param values new values + * @return execute result + * @throws Exception exception + */ + public ObPayload putWithResult(final String tableName, final Row rowKey, + final List keyRanges, final String[] columns, + final Object[] values) throws Exception { + final long start = System.currentTimeMillis(); + return executeMutation(tableName, + new MutationExecuteCallback(rowKey, keyRanges) { + /** + * Execute. + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, PUT, rowKey.getValues(), columns, values, + obTable.getObTableOperationTimeout()); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "PUT", endpoint, rowKey.getValues(), + (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + + /** + * Replace. + */ + public Replace replace(String tableName) { + return new Replace(this, tableName); + } + /** * Replace. */ @Override public long replace(final String tableName, final Object[] rowKey, final String[] columns, final Object[] values) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + final long start = System.currentTimeMillis(); return execute(tableName, new TableExecuteCallback(rowKey) { /** * Execute. */ @Override - public Long execute(ObPair obPair) throws Exception { - ObTable obTable = obPair.getRight(); - long partId = obPair.getLeft(); + public Long execute(ObPair obPair) throws Exception { + long getTableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); ObTableOperationRequest request = ObTableOperationRequest.getInstance(tableName, REPLACE, rowKey, columns, values, obTable.getObTableOperationTimeout()); - request.setPartitionId(partId); - ObPayload result = obPair.getRight().execute(request); - checkObTableOperationResult(obTable.getIp(), obTable.getPort(), result); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "REPLACE", endpoint, rowKey, + (ObTableOperationResult) result, getTableTime - start, + System.currentTimeMillis() - getTableTime, getslowQueryMonitorThreshold()); + checkObTableOperationResult(obTable.getIp(), obTable.getPort(), request, result); return ((ObTableOperationResult) result).getAffectedRows(); } }); } + /** + * Replace with result + * @param tableName which table to replace + * @param rowKey replace row key + * @param keyRanges scan range + * @param columns columns name to replace + * @param values new values + * @return execute result + * @throws Exception exception + */ + public ObPayload replaceWithResult(final String tableName, final Row rowKey, + final List keyRanges, final String[] columns, + final Object[] values) throws Exception { + final long start = System.currentTimeMillis(); + return executeMutation(tableName, + new MutationExecuteCallback(rowKey, keyRanges) { + /** + * Execute. + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, REPLACE, rowKey.getValues(), columns, values, + obTable.getObTableOperationTimeout()); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "REPLACE", endpoint, rowKey.getValues(), + (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + + /** + * Insert or update. + */ + public InsertOrUpdate insertOrUpdate(String tableName) { + return new InsertOrUpdate(this, tableName); + } + /** * Insert or update. */ @Override public long insertOrUpdate(final String tableName, final Object[] rowKey, final String[] columns, final Object[] values) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + final long start = System.currentTimeMillis(); return execute(tableName, new TableExecuteCallback(rowKey) { /** * Execute. */ @Override - public Long execute(ObPair obPair) throws Exception { - ObTable obTable = obPair.getRight(); - long partId = obPair.getLeft(); + public Long execute(ObPair obPair) throws Exception { + long getTableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); ObTableOperationRequest request = ObTableOperationRequest.getInstance(tableName, INSERT_OR_UPDATE, rowKey, columns, values, obTable.getObTableOperationTimeout()); - request.setPartitionId(partId); - ObPayload result = obPair.getRight().execute(request); - checkObTableOperationResult(obTable.getIp(), obTable.getPort(), result); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "INERT_OR_UPDATE", endpoint, rowKey, + (ObTableOperationResult) result, getTableTime - start, + System.currentTimeMillis() - getTableTime, getslowQueryMonitorThreshold()); + checkObTableOperationResult(obTable.getIp(), obTable.getPort(), request, result); return ((ObTableOperationResult) result).getAffectedRows(); } }); } + /** + * InsertOrUpdate with result + * @param tableName which table to InsertOrUpdate + * @param rowKey InsertOrUpdate row key + * @param keyRanges scan range + * @param columns columns name to InsertOrUpdate + * @param values new values + * @param usePut use put or not + * @return execute result + * @throws Exception exception + */ + public ObPayload insertOrUpdateWithResult(final String tableName, final Row rowKey, + final List keyRanges, + final String[] columns, final Object[] values, + boolean usePut) throws Exception { + final long start = System.currentTimeMillis(); + return executeMutation(tableName, + new MutationExecuteCallback(rowKey, keyRanges) { + /** + * Execute. + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, INSERT_OR_UPDATE, rowKey.getValues(), columns, values, + obTable.getObTableOperationTimeout()); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + if (usePut) { + request.setOptionFlag(ObTableOptionFlag.USE_PUT); + } + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "INERT_OR_UPDATE", endpoint, + rowKey.getValues(), (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + + /** + * Put. + */ + public Put put(String tableName) { + return new Put(this, tableName); + } + + /** + * Increment. + */ + public Increment increment(String tableName) { + return new Increment(this, tableName); + } + + /** + * + * @param tableName which table to increment + * @param rowKey increment row key + * @param columns columns name to increment + * @param values new valuess + * @param withResult whether to bring back result + * @return execute result + * @throws Exception exception + */ @Override public Map increment(final String tableName, final Object[] rowKey, final String[] columns, final Object[] values, final boolean withResult) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + final long start = System.currentTimeMillis(); return execute(tableName, new TableExecuteCallback>(rowKey) { + /** + * + * @param obPair + * @return + * @throws Exception + */ @Override - public Map execute(ObPair obPair) throws Exception { - ObTable obTable = obPair.getRight(); - long partId = obPair.getLeft(); + public Map execute(ObPair obPair) throws Exception { + long getTableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); ObTableOperationRequest request = ObTableOperationRequest.getInstance(tableName, INCREMENT, rowKey, columns, values, obTable.getObTableOperationTimeout()); request.setReturningAffectedEntity(withResult); - request.setPartitionId(partId); - ObPayload result = obPair.getRight().execute(request); - checkObTableOperationResult(obTable.getIp(), obTable.getPort(), result); + request.setTableId(tableParam.getTableId()); + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "INCREMENT", endpoint, rowKey, + (ObTableOperationResult) result, getTableTime - start, + System.currentTimeMillis() - getTableTime, getslowQueryMonitorThreshold()); + checkObTableOperationResult(obTable.getIp(), obTable.getPort(), request, result); return ((ObTableOperationResult) result).getEntity().getSimpleProperties(); } }); } + /** + * Increment with result + * @param tableName which table to increment + * @param rowKey increment row key + * @param keyRanges scan range + * @param columns columns name to increment + * @param values new values + * @param withResult whether to bring back result + * @return execute result + * @throws Exception exception + */ + public ObPayload incrementWithResult(final String tableName, final Row rowKey, + final List keyRanges, final String[] columns, + final Object[] values, final boolean withResult) + throws Exception { + final long start = System.currentTimeMillis(); + return executeMutation(tableName, + new MutationExecuteCallback(rowKey, keyRanges) { + /** + * + * @param obPair + * @return + * @throws Exception + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, INCREMENT, rowKey.getValues(), columns, values, + obTable.getObTableOperationTimeout()); + request.setReturningAffectedEntity(withResult); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "INCREMENT", endpoint, rowKey.getValues(), + (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + + /** + * Append. + */ + public Append append(String tableName) { + return new Append(this, tableName); + } + @Override public Map append(final String tableName, final Object[] rowKey, final String[] columns, final Object[] values, final boolean withResult) throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + final long start = System.currentTimeMillis(); return execute(tableName, new TableExecuteCallback>(rowKey) { @Override - public Map execute(ObPair obPair) throws Exception { - ObTable obTable = obPair.getRight(); - long partId = obPair.getLeft(); + public Map execute(ObPair obPair) throws Exception { + long getTableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); ObTableOperationRequest request = ObTableOperationRequest.getInstance(tableName, APPEND, rowKey, columns, values, obTable.getObTableOperationTimeout()); request.setReturningAffectedEntity(withResult); - request.setPartitionId(partId); - ObPayload result = obPair.getRight().execute(request); - checkObTableOperationResult(obTable.getIp(), obTable.getPort(), result); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "INCREMENT", endpoint, rowKey, + (ObTableOperationResult) result, getTableTime - start, + System.currentTimeMillis() - getTableTime, getslowQueryMonitorThreshold()); + checkObTableOperationResult(obTable.getIp(), obTable.getPort(), request, result); return ((ObTableOperationResult) result).getEntity().getSimpleProperties(); } }); } + /** + * Append with result + * @param tableName which table to append + * @param rowKey append row key + * @param keyRanges scan range + * @param columns columns name to append + * @param values new values + * @param withResult whether to bring back row result + * @return execute result + * @throws Exception exception + */ + public ObPayload appendWithResult(final String tableName, final Row rowKey, + final List keyRanges, final String[] columns, + final Object[] values, final boolean withResult) + throws Exception { + final long start = System.currentTimeMillis(); + return executeMutation(tableName, + new MutationExecuteCallback(rowKey, keyRanges) { + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableOperationRequest request = ObTableOperationRequest.getInstance( + tableName, APPEND, rowKey.getValues(), columns, values, + obTable.getObTableOperationTimeout()); + request.setReturningAffectedEntity(withResult); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableName, "INCREMENT", endpoint, rowKey.getValues(), + (ObTableOperationResult) result, TableTime - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + + /** + * batch mutation. + */ + public BatchOperation batchOperation(String tableName) { + return new BatchOperation(this, tableName); + } + + /** + * execute mutation with filter + * @param tableQuery table query + * @param rowKey row key which want to mutate + * @param keyRanges scan range + * @param operation table operation + * @param withResult whether to bring back result + * @return execute result + * @throws Exception exception + */ + public ObPayload mutationWithFilter(final TableQuery tableQuery, final Row rowKey, + final List keyRanges, + final ObTableOperation operation, final boolean withResult) + throws Exception { + return mutationWithFilter(tableQuery, rowKey, keyRanges, operation, withResult, false, + false); + } + + /** + * execute mutation with filter + * @param tableQuery table query + * @param rowKey row key which want to mutate + * @param keyRanges scan range + * @param operation table operation + * @param withResult whether to bring back result + * @param checkAndExecute whether execute check and execute instead of query and mutate + * @param checkExists whether to check exists or not + * @return execute result + * @throws Exception exception + */ + public ObPayload mutationWithFilter(final TableQuery tableQuery, final Row rowKey, + final List keyRanges, + final ObTableOperation operation, final boolean withResult, + final boolean checkAndExecute, final boolean checkExists) + throws Exception { + final long start = System.currentTimeMillis(); + if (tableQuery != null && tableQuery.getObTableQuery().getKeyRanges().isEmpty()) { + // fill a whole range if no range is added explicitly. + tableQuery.getObTableQuery().addKeyRange(ObNewRange.getWholeRange()); + } + return executeMutation(tableQuery.getTableName(), new MutationExecuteCallback( + rowKey, keyRanges) { + /** + * Execute. + */ + @Override + public ObPayload execute(ObPair obPair) throws Exception { + long TableTime = System.currentTimeMillis(); + ObTableParam tableParam = obPair.getRight(); + ObTable obTable = tableParam.getObTable(); + ObTableQueryAndMutateRequest request = obTableQueryAndMutate(operation, tableQuery, + false); + request.setTimeout(obTable.getObTableOperationTimeout()); + request.setReturningAffectedEntity(withResult); + request.setTableId(tableParam.getTableId()); + // partId/tabletId + request.setPartitionId(tableParam.getPartitionId()); + request.getTableQueryAndMutate().setIsCheckAndExecute(checkAndExecute); + request.getTableQueryAndMutate().setIsCheckNoExists(!checkExists); + ObPayload result = obTable.execute(request); + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MonitorUtil.info(request, database, tableQuery.getTableName(), "QUERY_AND_MUTATE", + operation.getOperationType().toString(), endpoint, + (ObTableQueryAndMutateResult) result, tableQuery.getObTableQuery(), TableTime + - start, + System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); + checkResult(obTable.getIp(), obTable.getPort(), request, result); + return result; + } + }); + } + + /** + * + * @param tableQuery table query + * @param columns columns name + * @param values new value + * @return mutate request + * @throws Exception exceotion + */ + public ObTableQueryAndMutateRequest obTableQueryAndUpdate(final TableQuery tableQuery, + final String[] columns, + final Object[] values) + throws Exception { + if (null == columns || null == values || 0 == columns.length || 0 == values.length) { + throw new ObTableException("client get unexpected empty columns or values"); + } + ObTableOperation operation = ObTableOperation.getInstance(UPDATE, new Object[] {}, columns, + values); + return obTableQueryAndMutate(operation, tableQuery, false); + } + + /** + * + * @param tableQuery table query + * @return delete request + * @throws Exception exception + */ + + public ObTableQueryAndMutateRequest obTableQueryAndDelete(final TableQuery tableQuery) + throws Exception { + ObTableOperation operation = ObTableOperation.getInstance(DEL, new Object[] {}, null, null); + return obTableQueryAndMutate(operation, tableQuery, false); + } + + /** + * + * @param tableQuery table query + * @param columns columns name + * @param values new values + * @param withResult whether to bring back result + * @return increment result + * @throws Exception exception + */ + public ObTableQueryAndMutateRequest obTableQueryAndIncrement(final TableQuery tableQuery, + final String[] columns, + final Object[] values, + final boolean withResult) + throws Exception { + if (null == columns || null == values || 0 == columns.length || 0 == values.length) { + throw new ObTableException("client get unexpected empty columns or values"); + } + ObTableOperation operation = ObTableOperation.getInstance(INCREMENT, new Object[] {}, + columns, values); + return obTableQueryAndMutate(operation, tableQuery, withResult); + } + + /** + * + * @param tableQuery table query + * @param columns columns name + * @param values new values + * @param withResult whether to bring back result + * @return append result + * @throws Exception exception + */ + public ObTableQueryAndMutateRequest obTableQueryAndAppend(final TableQuery tableQuery, + final String[] columns, + final Object[] values, + final boolean withResult) + throws Exception { + if (null == columns || null == values || 0 == columns.length || 0 == values.length) { + throw new ObTableException("client get unexpected empty columns or values"); + } + ObTableOperation operation = ObTableOperation.getInstance(APPEND, new Object[] {}, columns, + values); + return obTableQueryAndMutate(operation, tableQuery, withResult); + } + + /** + * + * @param operation table operation + * @param tableQuery table query + * @param withResult whether to bring back result + * @return + * @throws Exception + */ + ObTableQueryAndMutateRequest obTableQueryAndMutate(final ObTableOperation operation, + final TableQuery tableQuery, + final boolean withResult) throws Exception { + ObTableQuery obTableQuery = tableQuery.getObTableQuery(); + String tableName = tableQuery.getTableName(); + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + + ObTableBatchOperation operations = new ObTableBatchOperation(); + + operations.addTableOperation(operation); + + ObTableQueryAndMutate queryAndMutate = buildObTableQueryAndMutate(obTableQuery, operations); + + ObTableQueryAndMutateRequest request = buildObTableQueryAndMutateRequest(queryAndMutate, + tableName); + + request.setOptionFlag(ObTableOptionFlag.DEFAULT); + request.setReturningAffectedEntity(withResult); + request.setReturningAffectedRows(true); + + return request; + } + /** * Execute. */ + /** + * Excute + * @param request request + * @return response + * @throws Exception if fail + */ public ObPayload execute(final ObTableAbstractOperationRequest request) throws Exception { + if (request.getTableName() == null || request.getTableName().isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } if (request instanceof ObTableOperationRequest) { ObTableBatchOperation batchOperation = new ObTableBatchOperation(); batchOperation.addTableOperation(((ObTableOperationRequest) request) @@ -1169,10 +2804,22 @@ public ObPayload execute(final ObTableAbstractOperationRequest request) throws E .executeInternal(); return batchOpsResult.getResults().get(0); } else if (request instanceof ObTableQueryRequest) { - ObTableClientQueryImpl tableQuery = new ObTableClientQueryImpl(request.getTableName(), + // TableGroup -> TableName + String tableName = request.getTableName(); + tableName = getPhyTableNameFromTableGroup(((ObTableQueryRequest) request), tableName); + ObTableClientQueryImpl tableQuery = new ObTableClientQueryImpl(tableName, ((ObTableQueryRequest) request).getTableQuery(), this); tableQuery.setEntityType(request.getEntityType()); return new ObClusterTableQuery(tableQuery).executeInternal(); + } else if (request instanceof ObTableQueryAsyncRequest) { + // TableGroup -> TableName + String tableName = request.getTableName(); + tableName = getPhyTableNameFromTableGroup( + ((ObTableQueryAsyncRequest) request).getObTableQueryRequest(), tableName); + ObTableClientQueryImpl tableQuery = new ObTableClientQueryImpl(tableName, + ((ObTableQueryAsyncRequest) request).getObTableQueryRequest().getTableQuery(), this); + tableQuery.setEntityType(request.getEntityType()); + return new ObClusterTableQuery(tableQuery).asyncExecuteInternal(); } else if (request instanceof ObTableBatchOperationRequest) { ObTableClientBatchOpsImpl batchOps = new ObTableClientBatchOpsImpl( request.getTableName(), @@ -1183,36 +2830,49 @@ public ObPayload execute(final ObTableAbstractOperationRequest request) throws E ObTableQueryAndMutate tableQueryAndMutate = ((ObTableQueryAndMutateRequest) request) .getTableQueryAndMutate(); ObTableQuery tableQuery = tableQueryAndMutate.getTableQuery(); - Map partIdMapObTable = new HashMap(); - for (ObNewRange rang : tableQuery.getKeyRanges()) { - ObRowKey startKey = rang.getStartKey(); - int startKeySize = startKey.getObjs().size(); - ObRowKey endKey = rang.getEndKey(); - int endKeySize = endKey.getObjs().size(); - Object[] start = new Object[startKeySize]; - Object[] end = new Object[endKeySize]; - for (int i = 0; i < startKeySize; i++) { - start[i] = startKey.getObj(i).getValue(); - } + // fill a whole range if no range is added explicitly. + if (tableQuery.getKeyRanges().isEmpty()) { + tableQuery.addKeyRange(ObNewRange.getWholeRange()); + } + if (isOdpMode()) { + request.setTimeout(getOdpTable().getObTableOperationTimeout()); + return getOdpTable().execute(request); + } else { + Map partIdMapObTable = new HashMap(); + for (ObNewRange rang : tableQuery.getKeyRanges()) { + ObRowKey startKey = rang.getStartKey(); + int startKeySize = startKey.getObjs().size(); + ObRowKey endKey = rang.getEndKey(); + int endKeySize = endKey.getObjs().size(); + Object[] start = new Object[startKeySize]; + Object[] end = new Object[endKeySize]; + for (int i = 0; i < startKeySize; i++) { + start[i] = startKey.getObj(i).getValue(); + } - for (int i = 0; i < endKeySize; i++) { - end[i] = endKey.getObj(i).getValue(); + for (int i = 0; i < endKeySize; i++) { + end[i] = endKey.getObj(i).getValue(); + } + ObBorderFlag borderFlag = rang.getBorderFlag(); + List> pairList = getTables(request.getTableName(), + tableQuery, start, borderFlag.isInclusiveStart(), end, + borderFlag.isInclusiveEnd(), false, false); + for (ObPair pair : pairList) { + partIdMapObTable.put(pair.getLeft(), pair.getRight()); + } } - ObBorderFlag borderFlag = rang.getBorderFlag(); - List> pairList = getTables(request.getTableName(), start, - borderFlag.isInclusiveStart(), end, borderFlag.isInclusiveEnd(), false, false); - for (ObPair pair : pairList) { - partIdMapObTable.put(pair.getLeft(), pair.getRight()); + if (partIdMapObTable.size() > 1) { + throw new ObTablePartitionConsistentException( + "query and mutate must be a atomic operation"); } - } - if (partIdMapObTable.size() > 1) { - throw new ObTablePartitionConsistentException( - "query and mutate must be a atomic operation"); - } - for (Long partId : partIdMapObTable.keySet()) { - request.setPartitionId(partId); - return partIdMapObTable.get(partId).execute(request); + for (Long partId : partIdMapObTable.keySet()) { + ObTableParam tableParam = partIdMapObTable.get(partId); + request.setTableId(tableParam.getTableId()); + request.setPartitionId(tableParam.getPartitionId()); + request.setTimeout(tableParam.getObTable().getObTableOperationTimeout()); + return tableParam.getObTable().execute(request); + } } } @@ -1220,15 +2880,45 @@ public ObPayload execute(final ObTableAbstractOperationRequest request) throws E + "is not supported. make sure the correct version"); } + private ObTableQueryAndMutate buildObTableQueryAndMutate(ObTableQuery obTableQuery, + ObTableBatchOperation obTableBatchOperation) { + ObTableQueryAndMutate queryAndMutate = new ObTableQueryAndMutate(); + queryAndMutate.setTableQuery(obTableQuery); + queryAndMutate.setMutations(obTableBatchOperation); + return queryAndMutate; + } + + private ObTableQueryAndMutateRequest buildObTableQueryAndMutateRequest(ObTableQueryAndMutate queryAndMutate, + String targetTableName) { + ObTableQueryAndMutateRequest request = new ObTableQueryAndMutateRequest(); + request.setTableName(targetTableName); + request.setTableQueryAndMutate(queryAndMutate); + request.setEntityType(ObTableEntityType.KV); + return request; + } + + /** + * checkAndInsUp. + */ + public CheckAndInsUp checkAndInsUp(String tableName, ObTableFilter filter, + InsertOrUpdate insUp, boolean checkExists) { + return new CheckAndInsUp(this, tableName, filter, insUp, checkExists); + } + /** - * Set full user name. + * Set full username + * @param fullUserName user name + * @throws IllegalArgumentException if userName invalid */ public void setFullUserName(String fullUserName) throws IllegalArgumentException { if (StringUtils.isBlank(fullUserName)) { + RUNTIME.error(String.format("full username is empty, full username=%s", fullUserName)); throw new IllegalArgumentException(String.format( "full username is empty, full username=%s", fullUserName)); } - if (-1 != fullUserName.indexOf('@') || -1 != fullUserName.indexOf('#')) { + if (this.odpMode == true) { + // do nothing, just pass raw username to odp + } else if (-1 != fullUserName.indexOf('@') || -1 != fullUserName.indexOf('#')) { parseStandardFullUsername(fullUserName); } else { parseNonStandardFullUsername(fullUserName); @@ -1238,7 +2928,7 @@ public void setFullUserName(String fullUserName) throws IllegalArgumentException /** * Set sys user name to access meta table. - * @param sysUserName + * @param sysUserName system user name */ public void setSysUserName(String sysUserName) { sysUA.setUserName(sysUserName); @@ -1246,7 +2936,7 @@ public void setSysUserName(String sysUserName) { /** * Set sys user password to access meta table. - * @param sysPassword + * @param sysPassword system password */ public void setSysPassword(String sysPassword) { sysUA.setPassword(sysPassword); @@ -1254,7 +2944,8 @@ public void setSysPassword(String sysPassword) { /** * Set sys user encrypted password to access meta table. - * @param encSysPassword + * @param encSysPassword encrypted system password + * @throws Exception if fail */ public void setEncSysPassword(String encSysPassword) throws Exception { sysUA.setEncPassword(encSysPassword); @@ -1266,8 +2957,12 @@ private void parseStandardFullUsername(String username) { utIndex = username.indexOf('@'); tcIndex = username.indexOf('#'); if (-1 == utIndex || -1 == tcIndex || utIndex >= tcIndex) { - throw new IllegalArgumentException(String.format("invalid full username, username=%s", - username)); + RUNTIME.error(String.format("invalid full username, username=%s", username)); + throw new IllegalArgumentException( + String + .format( + "invalid full username, username=%s (which should be userName@tenantName#clusterName)", + username)); } String user = username.substring(0, utIndex); @@ -1278,6 +2973,8 @@ private void parseStandardFullUsername(String username) { private void parseNonStandardFullUsername(String username) { if (StringUtils.isBlank(usernameSeparators)) { + RUNTIME.error(String.format( + "non standard username separators has not been set, full username=%s", username)); throw new IllegalArgumentException(String.format( "non standard username separators has not been set, full username=%s", username)); } @@ -1295,6 +2992,8 @@ private void parseNonStandardFullUsername(String username) { } if (-1 == ctIndex || -1 == tuIndex || (ctIndex == tuIndex)) { + RUNTIME.error(String.format("invalid full username, username=%s, userSeparators=%s", + username, usernameSeparators)); throw new IllegalArgumentException(String.format( "invalid full username, username=%s, userSeparators=%s", username, usernameSeparators)); @@ -1306,16 +3005,26 @@ private void parseNonStandardFullUsername(String username) { handleFullUsername(user, tenant, cluster, username); } + /** + * + * @param user + * @param tenant + * @param cluster + * @param username + */ private void handleFullUsername(String user, String tenant, String cluster, String username) { if (StringUtils.isBlank(user)) { + RUNTIME.error(String.format("user has not been set, username=%s", username)); throw new IllegalArgumentException(String.format("user has not been set, username=%s", username)); } if (StringUtils.isBlank(tenant)) { + RUNTIME.error(String.format("tenant has not been set, username=%s", username)); throw new IllegalArgumentException(String.format( "tenant has not been set, username=%s", username)); } if (StringUtils.isBlank(cluster)) { + RUNTIME.error(String.format("cluster has not been set, username=%s", username)); throw new IllegalArgumentException(String.format( "cluster has not been set, username=%s", username)); } @@ -1325,24 +3034,27 @@ private void handleFullUsername(String user, String tenant, String cluster, Stri } /** - * Get param u r l. + * Get param url + * @return param url */ public String getParamURL() { return paramURL; } - /* - * Config URL must contain database and may contain read_consistency - * example: - * http://configserver.com/services?Action=ObRootServiceInfo&User_ID=xxx&UID=uuu&ObRegion=rrr - * &read_consitency=weak&database=test + /** + * Set param url. + * @param paramURL param url + * @throws IllegalArgumentException if paramURL invalid */ public void setParamURL(String paramURL) throws IllegalArgumentException { if (StringUtils.isBlank(paramURL)) { + RUNTIME.error(String.format("zdal url is empty, url=%s", paramURL)); throw new IllegalArgumentException(String.format("zdal url is empty, url=%s", paramURL)); } int paramIndex = paramURL.indexOf('?'); if (-1 == paramIndex || (paramIndex + 1) == paramURL.length()) { + RUNTIME.error(String.format("invalid zdal url, parameters are not set. url=%s", + paramURL)); throw new IllegalArgumentException(String.format( "invalid zdal url, parameters are not set. url=%s", paramURL)); } @@ -1353,57 +3065,63 @@ public void setParamURL(String paramURL) throws IllegalArgumentException { for (String param : params) { String kv[] = param.split("="); if (2 != kv.length) { + RUNTIME.error(String.format("invalid parameter format. url=%s", paramURL)); throw new IllegalArgumentException(String.format( "invalid parameter format. url=%s", paramURL)); } if (Constants.DATABASE.equalsIgnoreCase(kv[0])) { db = kv[1]; - if (logger.isInfoEnabled()) { - logger.info(String.format("will set database=%s", kv[1])); + if (BOOT.isInfoEnabled()) { + BOOT.info(String.format("will set database=%s", kv[1])); } } else if (Constants.READ_CONSISTENCY.equalsIgnoreCase(kv[0])) { readConsistency = ObReadConsistency.getByName(kv[1]); - if (logger.isInfoEnabled()) { - logger.info(String.format("will set %s=%s", Constants.READ_CONSISTENCY, kv[1])); + if (BOOT.isInfoEnabled()) { + BOOT.info(String.format("will set %s=%s", Constants.READ_CONSISTENCY, kv[1])); } } else if (Constants.OB_ROUTE_POLICY.equalsIgnoreCase(kv[0])) { obRoutePolicy = ObRoutePolicy.getByName(kv[1]); - if (logger.isInfoEnabled()) { - logger.info(String.format("will set %s=%s", Constants.OB_ROUTE_POLICY, kv[1])); + if (BOOT.isInfoEnabled()) { + BOOT.info(String.format("will set %s=%s", Constants.OB_ROUTE_POLICY, kv[1])); } } } if (StringUtils.isBlank(db)) { - throw new IllegalArgumentException(String.format("database is empty. url=%s", paramURL)); + throw new IllegalArgumentException(String.format( + "database is empty in paramURL(configURL). url=%s", paramURL)); } setDatabase(db); this.paramURL = paramURL; } /** - * Get full user name. + * Get full username + * @return user name */ public String getFullUserName() { return fullUserName; } /** - * Get user name. + * Get username + * @return username */ public String getUserName() { return userName; } /** - * Set user name. + * Set username + * @param userName username */ public void setUserName(String userName) { this.userName = userName; } /** - * Get tenant name. + * Get tenant name + * @return tenant name */ public String getTenantName() { return tenantName; @@ -1411,62 +3129,71 @@ public String getTenantName() { /** * Set tenant name. + * @param tenantName tenant name */ public void setTenantName(String tenantName) { this.tenantName = tenantName; } /** - * Get cluster name. + * Get cluster name + * @return ob cluster name */ public String getClusterName() { return clusterName; } /** - * Set cluster name. + * Set cluster name + * @param clusterName ob cluster name */ public void setClusterName(String clusterName) { this.clusterName = clusterName; } /** - * Get password. + * Get password + * @return password */ public String getPassword() { return password; } /** - * Set password. + * Set password + * @param password password */ public void setPassword(String password) { this.password = password; } /** - * Get database. + * Get database + * @return database */ public String getDatabase() { return database; } /** - * Set database. + * Set database + * @param database database */ public void setDatabase(String database) { this.database = database; } /** - * Get data source name. + * Get data source name + * @return data source name */ public String getDataSourceName() { return dataSourceName; } /** - * Set data source name. + * Set data source name + * @param dataSourceName data source name */ public void setDataSourceName(String dataSourceName) { this.dataSourceName = dataSourceName; @@ -1474,6 +3201,7 @@ public void setDataSourceName(String dataSourceName) { /** * Is retry on change master times. + * @return is retry */ public boolean isRetryOnChangeMasterTimes() { return retryOnChangeMasterTimes; @@ -1481,39 +3209,57 @@ public boolean isRetryOnChangeMasterTimes() { /** * Set retry on change master times. + * @param retryOnChangeMasterTimes set retry */ public void setRetryOnChangeMasterTimes(boolean retryOnChangeMasterTimes) { this.retryOnChangeMasterTimes = retryOnChangeMasterTimes; } /** - * Add row key element. + * Add row key element + * @param tableName table name + * @param columns rowkey columns */ public void addRowKeyElement(String tableName, String[] columns) { if (columns == null || columns.length == 0) { + RUNTIME.error("add row key element error table " + tableName + " column " + + Arrays.toString(columns)); throw new IllegalArgumentException("add row key element error table " + tableName + " column " + Arrays.toString(columns)); } - Map rowKeyElement = new HashMap(); + if (tableName == null || tableName.length() == 0) { + throw new IllegalArgumentException("table name is null"); + } + Map rowKeyElement = new LinkedHashMap(); for (int i = 0; i < columns.length; i++) { rowKeyElement.put(columns[i], i); } tableRowKeyElement.put(tableName, rowKeyElement); } + public Map getRowKeyElement(String tableName) { + return tableRowKeyElement.get(tableName); + } + /** * Set running mode. + * @param runningMode mode, NORMAL: table client, HBASE: hbase client. */ public void setRunningMode(RunningMode runningMode) { this.runningMode = runningMode; } + public RunningMode getRunningMode() { + return this.runningMode; + } + public enum RunningMode { NORMAL, HBASE; } /** - * Get Read Consistency. + * Get read consistency. + * @return read consistency level. */ public ObReadConsistency getReadConsistency() { ObReadConsistency readConsistency = ThreadLocalMap.getReadConsistency(); @@ -1524,16 +3270,21 @@ public ObReadConsistency getReadConsistency() { } /** - * Get OB route policy. + * Get OB router policy. + * @return policy */ public ObRoutePolicy getObRoutePolicy() { return obRoutePolicy; } /** - * Get read route. + * Get OB router. + * @return router */ public ObServerRoute getReadRoute() { + if (odpMode) { + return null; + } if (getReadConsistency().isStrong()) { return STRONG_READ; } @@ -1543,6 +3294,8 @@ public ObServerRoute getReadRoute() { /** * Get route for read or write. + * @param readonly is readonly + * @return route */ public ObServerRoute getRoute(boolean readonly) { if (readonly) { @@ -1552,8 +3305,17 @@ public ObServerRoute getRoute(boolean readonly) { } } + public void setOdpAddr(String odpAddr) { + this.odpAddr = odpAddr; + } + + public void setOdpPort(int odpPort) { + this.odpPort = odpPort; + } + /** * Set current IDC, for testing only. + * @param idc idc */ public void setCurrentIDC(String idc) { this.currentIDC = idc; @@ -1567,4 +3329,246 @@ public String toString() { + ", \n ocpModel = " + ocpModel + "\n}\n"; } + public ConcurrentHashMap getTableGroupInverted() { + return TableGroupInverted; + } + + public ConcurrentHashMap getTableGroupCache() { + return TableGroupCache; + } + + /** + * get table route fail than clear table group message + * @param tableGroupName table group name that need to delete + */ + public void eraseTableGroupFromCache(String tableGroupName) { + // clear table group cache + TableGroupInverted.remove(TableGroupCache.get(tableGroupName)); + TableGroupCache.remove(tableGroupName); + TableGroupCacheLocks.remove(tableGroupName); + } + + /* + * check table name whether group name + */ + public boolean isTableGroupName(String tabName) { + return !tabName.contains("$"); + } + + /* + * get phy table name form table group + * if odp mode then do nothing + */ + public String getPhyTableNameFromTableGroup(ObTableQueryRequest request, String tableName) + throws Exception { + if (odpMode) { + // do nothing + } else if (request.getTableQuery().isHbaseQuery() && isTableGroupName(tableName)) { + tableName = tryGetTableNameFromTableGroupCache(tableName, false); + } + return tableName; + } + + /* + * Get the start keys of different tablets, byte[0] = [] = EMPTY_START_ROW = EMPTY_END_ROW + * Example: + * For Non Partition: return [[[]]] + * For Key Partition: return [[[]]] + * For Hash Partition: return [[[]]] + * For Range Partition: return [[[], [], []], ['a', [], []], ['z', 'b', 'c']] + */ + public byte[][][] getFirstPartStartKeys(String tableName) throws Exception { + // Check row key element + // getOrRefreshTableEntry() need row key element, we could remove this after we remove rk element + if (this.runningMode != RunningMode.HBASE + && !this.tableRowKeyElement.containsKey(tableName)) { + throw new IllegalArgumentException("Row key element is empty for " + tableName); + } + + // Get the latest table entry + TableEntry tableEntry = getOrRefreshTableEntry(tableName, true, false, false); + + // Define start keys + byte[][][] firstPartStartKeys = new byte[0][][]; + + if (tableEntry.isPartitionTable()) { + if (null != tableEntry.getPartitionInfo()) { + if (null != tableEntry.getPartitionInfo().getFirstPartDesc()) { + ObPartFuncType obPartFuncType = tableEntry.getPartitionInfo() + .getFirstPartDesc().getPartFuncType(); + if (obPartFuncType.isRangePart()) { + // Range Part + ObRangePartDesc rangePartDesc = (ObRangePartDesc) tableEntry + .getPartitionInfo().getFirstPartDesc(); + List> highBoundVals = rangePartDesc.getHighBoundValues(); + int startKeysLen = highBoundVals.size(); + int partKeyLen = highBoundVals.get(0).size(); + firstPartStartKeys = new byte[startKeysLen][][]; + + // Init start keys + firstPartStartKeys[0] = new byte[partKeyLen][]; + for (int i = 0; i < partKeyLen; i++) { + firstPartStartKeys[0][i] = new byte[0]; + } + + // Fulfill other start keys + for (int i = 0; i < startKeysLen - 1; i++) { + List innerList = highBoundVals.get(i); + firstPartStartKeys[i + 1] = new byte[innerList.size()][]; + for (int j = 0; j < innerList.size(); j++) { + firstPartStartKeys[i + 1][j] = innerList.get(j); + } + } + } else { + // Key / Hash Part + ObPartDesc partDesc = tableEntry.getPartitionInfo().getFirstPartDesc(); + int partKeyLen = partDesc.getPartColumns().size(); + + // Init start keys + firstPartStartKeys = new byte[1][partKeyLen][]; + Arrays.fill(firstPartStartKeys[0], new byte[0]); + } + } + } + } else { + // Non partition table + firstPartStartKeys = new byte[1][1][]; + Arrays.fill(firstPartStartKeys[0], new byte[0]); + } + + return firstPartStartKeys; + } + + /* + * Get the start keys of different tablets, byte[0] = [] = EMPTY_START_ROW = EMPTY_BYTE_ARRAY + * Example: + * For Key Partition: return [[[]]] + * For Hash Partition: return [[[]]] + * For Range Partition: return [['a', [], []], ['z', 'b', 'c'], [[], [], []]] + */ + public byte[][][] getFirstPartEndKeys(String tableName) throws Exception { + // Check row key element + // getOrRefreshTableEntry() need row key element, we could remove this after we remove rk element + if (this.runningMode != RunningMode.HBASE && this.tableRowKeyElement.containsKey(tableName)) { + throw new IllegalArgumentException("Row key element is empty for " + tableName); + } + + // Get the latest table entry + TableEntry tableEntry = getOrRefreshTableEntry(tableName, true, false, false); + + // Define end keys + byte[][][] firstPartEndKeys = new byte[0][][]; + + if (tableEntry.isPartitionTable()) { + if (null != tableEntry.getPartitionInfo()) { + if (null != tableEntry.getPartitionInfo().getFirstPartDesc()) { + ObPartFuncType obPartFuncType = tableEntry.getPartitionInfo() + .getFirstPartDesc().getPartFuncType(); + if (obPartFuncType.isRangePart()) { + // Range Part + ObRangePartDesc rangePartDesc = (ObRangePartDesc) tableEntry + .getPartitionInfo().getFirstPartDesc(); + List> highBoundVals = rangePartDesc.getHighBoundValues(); + int endKeysLen = highBoundVals.size(); + firstPartEndKeys = new byte[endKeysLen][][]; + + // Fulfill end keys + for (int i = 0; i < endKeysLen; i++) { + List innerList = highBoundVals.get(i); + firstPartEndKeys[i] = new byte[innerList.size()][]; + for (int j = 0; j < innerList.size(); j++) { + firstPartEndKeys[i][j] = innerList.get(j); + } + } + } else { + // Key / Hash Part + ObPartDesc partDesc = tableEntry.getPartitionInfo().getFirstPartDesc(); + int partKeyLen = partDesc.getPartColumns().size(); + + // Init end keys + firstPartEndKeys = new byte[1][partKeyLen][]; + Arrays.fill(firstPartEndKeys[0], new byte[0]); + } + } + } + } else { + // Non partition table + firstPartEndKeys = new byte[1][1][]; + Arrays.fill(firstPartEndKeys[0], new byte[0]); + } + + return firstPartEndKeys; + } + + /* + * Get the start keys of HBase table + * Example: + * For Non Partition: return [EMPTY_START_ROW] + * For Key Partition: return [EMPTY_START_ROW] + * For Hash Partition: return [EMPTY_START_ROW] + * For Range Partition: return [[EMPTY_START_ROW, EMPTY_START_ROW, EMPTY_START_ROW], ['a', [], []], ['z', 'b', 'c']] + */ + public byte[][] getHBaseTableStartKeys(String hbaseTableName) throws Exception { + // Check HBase client + if (this.runningMode != RunningMode.HBASE) { + throw new IllegalArgumentException("getHBaseTableStartKeys only support in HBase mode"); + } + + // Get actual table name + String tableName = tryGetTableNameFromTableGroupCache(hbaseTableName, false); + + // Get start keys of first partition + byte[][][] firstPartStartKeys = getFirstPartStartKeys(tableName); + + // Result start keys + byte[][] startKeys = new byte[firstPartStartKeys.length][]; + + // Construct result keys + for (int i = 0; i < firstPartStartKeys.length; i++) { + if (firstPartStartKeys[i] == null || firstPartStartKeys[i].length != 1 + || firstPartStartKeys[i][0] == null || firstPartStartKeys[i][0].length > 1) { + throw new IllegalArgumentException("Invalid start keys structure at index " + i + + " for table " + hbaseTableName); + } + startKeys[i] = firstPartStartKeys[i][0]; + } + + return startKeys; + } + + /* + * Get the start keys of HBase table + * Example: + * For Non Partition: return [EMPTY_END_ROW] + * For Key Partition: return [EMPTY_END_ROW] + * For Hash Partition: return [EMPTY_END_ROW] + * For Range Partition: return [['a', [], []], ['z', 'b', 'c'], [EMPTY_END_ROW, EMPTY_END_ROW, EMPTY_END_ROW]] + */ + public byte[][] getHBaseTableEndKeys(String hbaseTableName) throws Exception { + // Check HBase client + if (this.runningMode != RunningMode.HBASE) { + throw new IllegalArgumentException("getHBaseTableStartKeys only support in HBase mode"); + } + + // Get actual table name + String tableName = tryGetTableNameFromTableGroupCache(hbaseTableName, false); + + // Get end keys of first partition + byte[][][] firstPartEndKeys = getFirstPartEndKeys(tableName); + + // Result end keys + byte[][] endKeys = new byte[firstPartEndKeys.length][]; + + // Construct result keys + for (int i = 0; i < firstPartEndKeys.length; i++) { + if (firstPartEndKeys[i] == null || firstPartEndKeys[i].length != 1 + || firstPartEndKeys[i][0] == null || firstPartEndKeys[i][0].length > 1) { + throw new IllegalArgumentException("Invalid end keys structure at index " + i + + " for table " + hbaseTableName); + } + endKeys[i] = firstPartEndKeys[i][0]; + } + + return endKeys; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/batch/QueryByBatch.java b/src/main/java/com/alipay/oceanbase/rpc/batch/QueryByBatch.java deleted file mode 100644 index 5357e040..00000000 --- a/src/main/java/com/alipay/oceanbase/rpc/batch/QueryByBatch.java +++ /dev/null @@ -1,259 +0,0 @@ -/*- - * #%L - * OBKV Table Client Framework - * %% - * Copyright (C) 2021 OceanBase - * %% - * OBKV Table Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.rpc.batch; - -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHTableFilter; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQuery; -import com.alipay.oceanbase.rpc.stream.QueryResultSet; -import com.alipay.oceanbase.rpc.table.api.TableQuery; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -public class QueryByBatch implements TableQuery { - private final TableQuery tableQuery; - private Object[] start, end; - private boolean endEquals; - private String[] keys = new String[0]; - private String[] select = new String[0]; - - public QueryByBatch(TableQuery tableQuery) { - this.tableQuery = tableQuery; - } - - @Override - public ObTableQuery getObTableQuery() { - return this.tableQuery.getObTableQuery(); - } - - @Override - public String getTableName() { - return this.tableQuery.getTableName(); - } - - @Override - public void setEntityType(ObTableEntityType entityType) { - this.tableQuery.setEntityType(entityType); - } - - @Override - public ObTableEntityType getEntityType() { - return this.tableQuery.getEntityType(); - } - - @Override - public QueryResultSet execute() { - return new QueryResultSet(new QueryByBatchResultSet(this)); - } - - public TableQuery addScanRange(Object start, Object end) { - addScanRange(start, true, end, true); - return this; - } - - @Override - public TableQuery addScanRange(Object[] start, Object[] end) { - return addScanRange(start, true, end, true); - } - - public TableQuery addScanRange(Object start, boolean startEquals, Object end, boolean endEquals) { - addScanRange(new Object[] { start }, startEquals, new Object[] { end }, endEquals); - return this; - } - - public TableQuery addScanRange(Object[] start, boolean startEquals, Object[] end, - boolean endEquals) { - this.start = start; - this.end = end; - this.endEquals = endEquals; - this.tableQuery.addScanRange(start, startEquals, end, endEquals); - return this; - } - - @Override - public TableQuery addScanRangeStartsWith(Object start) { - addScanRangeStartsWith(new Object[] { start }); - return this; - } - - @Override - public TableQuery addScanRangeStartsWith(Object[] start) { - addScanRangeStartsWith(start, true); - return this; - } - - @Override - public TableQuery addScanRangeStartsWith(Object[] start, boolean startEquals) { - this.start = start; - this.tableQuery.addScanRangeStartsWith(start, startEquals); - return this; - } - - @Override - public TableQuery addScanRangeEndsWith(Object end) { - addScanRangeEndsWith(new Object[] { end }); - return this; - } - - @Override - public TableQuery addScanRangeEndsWith(Object[] end) { - addScanRangeEndsWith(end, true); - return this; - } - - @Override - public TableQuery addScanRangeEndsWith(Object[] end, boolean endEquals) { - this.end = end; - this.endEquals = endEquals; - this.tableQuery.addScanRangeEndsWith(end, endEquals); - return this; - } - - @Override - public TableQuery scanOrder(boolean forward) { - throw new IllegalArgumentException("not support scanOrder for QueryByBatch"); - } - - @Override - public TableQuery indexName(String indexName) { - throw new IllegalArgumentException("not support indexName for QueryByBatch"); - } - - @Override - public TableQuery primaryIndex() { - throw new IllegalArgumentException("not support primaryIndex for QueryByBatch"); - } - - @Override - public TableQuery filterString(String filterString) { - throw new IllegalArgumentException("not support filterString for QueryByBatch"); - } - - @Override - public TableQuery setHTableFilter(ObHTableFilter obHTableFilter) { - throw new IllegalArgumentException("not support setHTableFilter for QueryByBatch"); - } - - /** - * 这边取名为 batch size, 实际转化到查询，就是stream的limit - * 保证它的查询只是单次的，所以在observer不会转化成stream query - * @param batchSize - * @return - */ - public QueryByBatch setBatchSize(int batchSize) { - this.tableQuery.limit(batchSize); - return this; - } - - @Override - public TableQuery setOperationTimeout(long operationTimeout) { - this.tableQuery.setOperationTimeout(operationTimeout); - return this; - } - - @Override - public void clear() { - this.tableQuery.clear(); - } - - public TableQuery getTableQuery() { - return tableQuery; - } - - /** - * 必须显式的设置一下 scan 的 keys - * 因为目前 observer 没有返回 primary key 或者其他 index 的信息 - * 所以拆分查询时，必须手动指定 key - * 需要和 select 的 key 合并 - * @param keys - * @return - */ - @Override - public TableQuery setKeys(String... keys) { - this.keys = keys; - mergeSelect(); - return this; - } - - @Override - public TableQuery limit(int limit) { - throw new IllegalArgumentException("not support limit for QueryByBatch"); - } - - @Override - public TableQuery limit(int offset, int limit) { - throw new IllegalArgumentException("not support limit for QueryByBatch"); - } - - /** - * 需要和 select 的 key 合并 - * @param columns - * @return - */ - @Override - public QueryByBatch select(String... columns) { - this.select = columns; - mergeSelect(); - return this; - } - - /** - * 只支持两种方式的合并 - * 1. keys: ["a"], select: ["b"], 合并为 ["a", "b"]，即不包含关系 - * 2. keys: ["a"], select: ["a", "b"]， 合并为 ["a", "b"] 即包含关系 - * 其他情况返回不支持 - */ - private void mergeSelect() { - Set keySet = new HashSet(Arrays.asList(this.keys)); - Set selectSet = new HashSet(Arrays.asList(this.select)); - if (keySet.size() != this.keys.length || selectSet.size() != this.select.length) { - throw new IllegalArgumentException("key duplicate in (keys or select)"); - } - Set mix = new HashSet(); - mix.addAll(keySet); - mix.addAll(selectSet); - if (mix.size() == keySet.size() + selectSet.size()) { - String[] mixSelect = new String[this.keys.length + this.select.length]; - System.arraycopy(this.keys, 0, mixSelect, 0, this.keys.length); - System.arraycopy(this.select, 0, mixSelect, this.keys.length, this.select.length); - this.tableQuery.select(mixSelect); - } else if (mix.size() == selectSet.size()) { - this.tableQuery.select(this.select); - } else { - throw new IllegalArgumentException("key duplicate in (keys and select)"); - } - - } - - public String[] getKeys() { - return keys; - } - - public Object[] getStart() { - return start; - } - - public Object[] getEnd() { - return end; - } - - public boolean isEndEquals() { - return endEquals; - } -} diff --git a/src/main/java/com/alipay/oceanbase/rpc/batch/QueryByBatchResultSet.java b/src/main/java/com/alipay/oceanbase/rpc/batch/QueryByBatchResultSet.java deleted file mode 100644 index 26820b97..00000000 --- a/src/main/java/com/alipay/oceanbase/rpc/batch/QueryByBatchResultSet.java +++ /dev/null @@ -1,120 +0,0 @@ -/*- - * #%L - * OBKV Table Client Framework - * %% - * Copyright (C) 2021 OceanBase - * %% - * OBKV Table Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.rpc.batch; - -import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.QueryStreamResult; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; -import com.alipay.oceanbase.rpc.stream.QueryResultSet; -import com.alipay.oceanbase.rpc.table.api.TableQuery; - -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -public class QueryByBatchResultSet implements QueryStreamResult { - private final QueryByBatch queryByBatch; - private QueryResultSet resultSet = null; - private int resultSize; - - public QueryByBatchResultSet(QueryByBatch queryByBatch) { - this.queryByBatch = queryByBatch; - } - - @Override - public boolean next() throws Exception { - // 如果是 null ，就说明它是第一次查询，直接使用老得接口 execute - if (resultSet == null) { - resultSet = this.queryByBatch.getTableQuery().execute(); - resultSize = resultSet.cacheSize(); - } - boolean res = resultSet.next(); - if (!res) { - // 防止出现 [0,0) 情况 - if (resultSize > 0) { - // 如果没有 next， getrow 获取的是上一个存在的值 - Map row = resultSet.getRow(); - updateQuery(row, this.queryByBatch.getTableQuery()); - refreshExecute(); - res = resultSet.next(); - } - } - return res; - } - - @Override - public List getRow() { - return resultSet.getQueryStreamResult().getRow(); - } - - /** - * 通过当前的最后一条数据，与之前的查询条件融合 - * 例如，上次查询为 limit["a","z"), 最后一条数据为 "bbb" - * 则此次查询改造为 limit("bbb", "z") - * @param row - * @param tableQuery - */ - private void updateQuery(Map row, TableQuery tableQuery) { - List ranges = tableQuery.getObTableQuery().getKeyRanges(); - ranges.clear(); - int keyLen = this.queryByBatch.getEnd().length; - Object[] newBegin = new Object[keyLen]; - for (int i = 0; i < keyLen; i++) { - newBegin[i] = row.get(this.queryByBatch.getKeys()[i]); - } - // 这边 startEquals 永远都是 false， 因为 begin 这条数据已经取过了 - tableQuery.addScanRange(newBegin, false, this.queryByBatch.getEnd(), - this.queryByBatch.isEndEquals()); - } - - /** - * 在更新完 query 以后，重新获取一下结果 - */ - private void refreshExecute() throws Exception { - resultSet = this.queryByBatch.getTableQuery().execute(); - resultSize = resultSet.cacheSize(); - } - - @Override - public int getRowIndex() { - return resultSet.getQueryStreamResult().getRowIndex(); - } - - @Override - public LinkedList> getCacheRows() { - if (resultSet == null) { - return new LinkedList>(); - } - return resultSet.getQueryStreamResult().getCacheRows(); - } - - @Override - public List getCacheProperties() { - return resultSet.getQueryStreamResult().getCacheProperties(); - } - - @Override - public void init() throws Exception { - resultSet.getQueryStreamResult().init(); - } - - @Override - public void close() throws Exception { - resultSet.getQueryStreamResult().close(); - } -} diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacket.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacket.java index 8626fc25..1dcf7d28 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacket.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacket.java @@ -54,7 +54,7 @@ public void decodePacketHeader() { this.setCmdCode(ObTablePacketCode.valueOf((short) header.getPcode())); } - /** + /* * Create transport error packet. */ public static ObTablePacket createTransportErrorPacket(int transportCode, String message, @@ -67,7 +67,7 @@ public static ObTablePacket createTransportErrorPacket(int transportCode, String return tablePacket; } - /** + /* * Get protocol code. */ @Override @@ -75,7 +75,7 @@ public ProtocolCode getProtocolCode() { return OB_RPC_MAGIC_CODE; } - /** + /* * Get cmd code. */ @Override @@ -83,14 +83,14 @@ public CommandCode getCmdCode() { return commandCode; } - /** + /* * Set cmd code. */ public void setCmdCode(CommandCode cmdCode) { this.commandCode = cmdCode; } - /** + /* * Get id. */ @Override @@ -98,108 +98,107 @@ public int getId() { return id; } - /** + /* * Set id. */ public void setId(int id) { this.id = id; } - /** + /* * Get packet content. */ public byte[] getPacketContent() { return packetContent; } - /** + /* * Set packet content. */ public void setPacketContent(byte[] packetContent) { this.packetContent = packetContent; } - /** + /* * Set packet content buf. */ public void setPacketContentBuf(ByteBuf packetContent) { this.packetContentBuf = packetContent; } - /** + /* * Get packet content buf. */ public ByteBuf getPacketContentBuf() { return packetContentBuf; } - /** + /* * Release byte buf. */ public void releaseByteBuf() { - // http://netty.io/wiki/reference-counted-objects.html if (packetContentBuf != null) { packetContentBuf.release(); } } - /** + /* * Get header. */ public ObRpcPacketHeader getHeader() { return header; } - /** + /* * Set header. */ public void setHeader(ObRpcPacketHeader header) { this.header = header; } - /** + /* * Is success. */ public boolean isSuccess() { return transportCode == 0; } - /** + /* * Get transport code. */ public int getTransportCode() { return transportCode; } - /** + /* * Set transport code. */ public void setTransportCode(int transportCode) { this.transportCode = transportCode; } - /** + /* * Get message. */ public String getMessage() { return message; } - /** + /* * Set message. */ public void setMessage(String message) { this.message = message; } - /** + /* * Get cause. */ public Throwable getCause() { return cause; } - /** + /* * Set cause. */ public void setCause(Throwable cause) { @@ -208,7 +207,7 @@ public void setCause(Throwable cause) { // TODO useless for now - /** + /* * Get invoke context. */ @Override @@ -216,7 +215,7 @@ public InvokeContext getInvokeContext() { return null; } - /** + /* * Get serializer. */ @Override @@ -224,7 +223,7 @@ public byte getSerializer() { return 0; } - /** + /* * Get protocol switch. */ @Override @@ -232,7 +231,7 @@ public ProtocolSwitch getProtocolSwitch() { return null; } - /** + /* * Serialize. */ @Override @@ -240,7 +239,7 @@ public void serialize() throws SerializationException { } - /** + /* * Deserialize. */ @Override @@ -248,7 +247,7 @@ public void deserialize() throws DeserializationException { } - /** + /* * Serialize content. */ @Override @@ -256,7 +255,7 @@ public void serializeContent(InvokeContext invokeContext) throws SerializationEx } - /** + /* * Deserialize content. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketCode.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketCode.java index 4aa73089..32228332 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketCode.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketCode.java @@ -17,20 +17,24 @@ package com.alipay.oceanbase.rpc.bolt.protocol; +import com.alipay.oceanbase.rpc.exception.ObTableRoutingWrongException; import com.alipay.oceanbase.rpc.protocol.packet.ObRpcPacketHeader; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableBatchOperationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableLSOpResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.login.ObTableLoginResult; import com.alipay.remoting.CommandCode; public enum ObTablePacketCode implements CommandCode { OB_TABLE_API_LOGIN(Pcodes.OB_TABLE_API_LOGIN) { - /** + /* * New payload. */ @Override @@ -39,7 +43,7 @@ public ObPayload newPayload(ObRpcPacketHeader header) { } }, // OB_TABLE_API_EXECUTE(Pcodes.OB_TABLE_API_EXECUTE) { - /** + /* * New payload. */ @Override @@ -48,7 +52,7 @@ public ObPayload newPayload(ObRpcPacketHeader header) { } }, // OB_TABLE_API_BATCH_EXECUTE(Pcodes.OB_TABLE_API_BATCH_EXECUTE) { - /** + /* * New payload. */ @Override @@ -57,7 +61,7 @@ public ObPayload newPayload(ObRpcPacketHeader header) { } }, // OB_TABLE_API_EXECUTE_QUERY(Pcodes.OB_TABLE_API_EXECUTE_QUERY) { - /** + /* * New payload. */ @Override @@ -66,7 +70,7 @@ public ObPayload newPayload(ObRpcPacketHeader header) { } }, // OB_TABLE_API_QUERY_AND_MUTATE(Pcodes.OB_TABLE_API_QUERY_AND_MUTATE) { - /** + /* * New payload. */ @Override @@ -74,13 +78,49 @@ public ObPayload newPayload(ObRpcPacketHeader header) { return new ObTableQueryAndMutateResult(); } }, // - - // INVALID REQUEST PCODE, no such rpc - OB_ERROR_PACKET(Pcodes.OB_ERROR_PACKET) { + OB_TABLE_API_EXECUTE_QUERY_SYNC(Pcodes.OB_TABLE_API_EXECUTE_QUERY_SYNC) { + /** + * New payload. + */ + @Override + public ObPayload newPayload(ObRpcPacketHeader header) { + return new ObTableQueryAsyncResult(); + } + }, // + OB_TABLE_API_DIRECT_LOAD(Pcodes.OB_TABLE_API_DIRECT_LOAD) { /** * New payload. */ @Override + public ObPayload newPayload(ObRpcPacketHeader header) { + return new ObTableDirectLoadResult(); + } + }, // + OB_TABLE_API_LS_EXECUTE(Pcodes.OB_TABLE_API_LS_EXECUTE) { + /** + * New payload. + */ + @Override + public ObPayload newPayload(ObRpcPacketHeader header) { + return new ObTableLSOpResult(); + } + }, // + OB_TABLE_API_MOVE(Pcodes.OB_TABLE_API_MOVE) { + /** + * New payload. + */ + @Override + public ObPayload newPayload(ObRpcPacketHeader header) { + throw new ObTableRoutingWrongException( + "Receive rerouting response packet. " + + "Java client is not supported and need to Refresh table router entry"); + } + }, // + OB_ERROR_PACKET(Pcodes.OB_ERROR_PACKET) { + /* + * New payload. + */ + @Override public ObPayload newPayload(ObRpcPacketHeader header) { throw new IllegalArgumentException("OB_ERROR_PACKET has no payload implementation"); } @@ -92,14 +132,14 @@ public ObPayload newPayload(ObRpcPacketHeader header) { this.value = (short) value; } - /** + /* * Value. */ public short value() { return this.value; } - /** + /* * Value of. */ public static ObTablePacketCode valueOf(short value) { @@ -114,6 +154,16 @@ public static ObTablePacketCode valueOf(short value) { return OB_TABLE_API_EXECUTE_QUERY; case Pcodes.OB_TABLE_API_QUERY_AND_MUTATE: return OB_TABLE_API_QUERY_AND_MUTATE; + case Pcodes.OB_TABLE_API_EXECUTE_QUERY_SYNC: + return OB_TABLE_API_EXECUTE_QUERY_SYNC; + case Pcodes.OB_TABLE_API_DIRECT_LOAD: + return OB_TABLE_API_DIRECT_LOAD; + case Pcodes.OB_TABLE_API_LS_EXECUTE: + return OB_TABLE_API_LS_EXECUTE; + case Pcodes.OB_TABLE_API_MOVE: + throw new ObTableRoutingWrongException( + "Receive rerouting response packet. " + + "Java client is not supported and need to Refresh table router entry"); case Pcodes.OB_ERROR_PACKET: return OB_ERROR_PACKET; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketDecoder.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketDecoder.java index 140e2893..78ca9d7f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketDecoder.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketDecoder.java @@ -26,7 +26,7 @@ public class ObTablePacketDecoder implements CommandDecoder { - /** + /* * Decode. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketEncoder.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketEncoder.java index 384f77a1..f20e984c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketEncoder.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTablePacketEncoder.java @@ -33,7 +33,7 @@ public class ObTablePacketEncoder implements CommandEncoder { private static final Logger logger = TableClientLoggerFactory .getLogger(ObTablePacketEncoder.class); - /** + /* * Encode. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTableProtocol.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTableProtocol.java index 5c7e0a4d..5506e6a5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTableProtocol.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/protocol/ObTableProtocol.java @@ -25,7 +25,7 @@ public class ObTableProtocol implements Protocol { public final static byte API_VERSION = 1; public final static int HEADER_LENGTH = 16; - /** + /* * ob_rpc_net_handler.cpp * Magic Header 4 bytes *

@@ -45,7 +45,7 @@ public class ObTableProtocol implements Protocol { ProtocolManager.registerProtocol(new ObTableProtocol(), MAGIC_HEADER_FLAG); } - /** + /* * Ob table protocol. */ public ObTableProtocol() { @@ -55,7 +55,7 @@ public ObTableProtocol() { this.commandHandler = new ObTablePacketHandler(); } - /** + /* * Get encoder. */ @Override @@ -63,7 +63,7 @@ public CommandEncoder getEncoder() { return commandEncoder; } - /** + /* * Get decoder. */ @Override @@ -71,7 +71,7 @@ public CommandDecoder getDecoder() { return commandDecoder; } - /** + /* * Get command handler. */ @Override @@ -79,7 +79,7 @@ public CommandHandler getCommandHandler() { return commandHandler; } - /** + /* * Get heartbeat trigger. */ @Override @@ -87,7 +87,7 @@ public HeartbeatTrigger getHeartbeatTrigger() { return null; } - /** + /* * Get command factory. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObClientFuture.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObClientFuture.java index 67ff0546..bf23b05f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObClientFuture.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObClientFuture.java @@ -22,39 +22,58 @@ import com.alipay.remoting.InvokeContext; import com.alipay.remoting.InvokeFuture; import com.alipay.remoting.RemotingCommand; +import com.alipay.oceanbase.rpc.exception.ObTableTimeoutExcetion; import io.netty.util.Timeout; import java.net.InetSocketAddress; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; public class ObClientFuture implements InvokeFuture { - private CountDownLatch waiter = new CountDownLatch(1); + private CountDownLatch waiter = new CountDownLatch(1); private RemotingCommand response; private int channelId; - /** + // BY_WORKER indicate response must be release by worker itself. + // BY_BACKGROUND indicate response must be release by background decoder thread + private static int INIT = 0; + private static int BY_WORKER = 1; + private static int BY_BACKGROUND = 2; + + private AtomicInteger releaseFlag = new AtomicInteger(INIT); + + /* * Ob client future. */ public ObClientFuture(int channelId) { this.channelId = channelId; } - /** + /* * Wait response. */ @Override public RemotingCommand waitResponse(long timeoutMillis) throws InterruptedException { - if (waiter.await(timeoutMillis, TimeUnit.MILLISECONDS)) { - return response; - } else { - return ObTablePacket.createTransportErrorPacket(TransportCodes.BOLT_TIMEOUT, - "wait timeout: " + timeoutMillis, null); + try { + if (waiter.await(timeoutMillis, TimeUnit.MILLISECONDS) + || !releaseFlag.compareAndSet(INIT, BY_BACKGROUND)) { + return response; + } else { + return null; + } + } catch (InterruptedException e) { + releaseFlag.set(BY_BACKGROUND); + if (response instanceof ObTablePacket) { + ((ObTablePacket) response).releaseByteBuf(); + } + throw e; + } finally { } } - /** + /* * Wait response. */ @Override @@ -63,16 +82,21 @@ public RemotingCommand waitResponse() throws InterruptedException { return response; } - /** + /* * Put response. */ @Override public void putResponse(RemotingCommand response) { this.response = response; waiter.countDown(); + if (!releaseFlag.compareAndSet(INIT, BY_WORKER)) { + if (response instanceof ObTablePacket) { + ((ObTablePacket) response).releaseByteBuf(); + } + } } - /** + /* * Invoke id. */ @Override @@ -80,7 +104,7 @@ public int invokeId() { return channelId; } - /** + /* * Is done. */ @Override @@ -88,7 +112,7 @@ public boolean isDone() { return this.waiter.getCount() == 0; } - /** + /* * Create connection closed response. */ @Override @@ -96,7 +120,7 @@ public RemotingCommand createConnectionClosedResponse(InetSocketAddress response return null; } - /** + /* * Execute invoke callback. */ @Override @@ -104,7 +128,7 @@ public void executeInvokeCallback() { } - /** + /* * Try async execute invoke callback abnormally. */ @Override @@ -112,7 +136,7 @@ public void tryAsyncExecuteInvokeCallbackAbnormally() { } - /** + /* * Set cause. */ @Override @@ -120,7 +144,7 @@ public void setCause(Throwable cause) { } - /** + /* * Get cause. */ @Override @@ -128,7 +152,7 @@ public Throwable getCause() { return null; } - /** + /* * Get invoke callback. */ @Override @@ -136,7 +160,7 @@ public InvokeCallback getInvokeCallback() { return null; } - /** + /* * Add timeout. */ @Override @@ -144,7 +168,7 @@ public void addTimeout(Timeout timeout) { } - /** + /* * Cancel timeout. */ @Override @@ -152,7 +176,7 @@ public void cancelTimeout() { } - /** + /* * Get app class loader. */ @Override @@ -160,7 +184,7 @@ public ClassLoader getAppClassLoader() { return null; } - /** + /* * Get protocol code. */ @Override @@ -168,7 +192,7 @@ public byte getProtocolCode() { return 0; } - /** + /* * Set invoke context. */ @Override @@ -176,7 +200,7 @@ public void setInvokeContext(InvokeContext invokeContext) { } - /** + /* * Get invoke context. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObConfigurableInstance.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObConfigurableInstance.java index 71672189..4fed909c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObConfigurableInstance.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObConfigurableInstance.java @@ -21,7 +21,7 @@ import com.alipay.remoting.config.configs.ConfigType; public class ObConfigurableInstance extends AbstractConfigurableInstance { - /** + /* * Ob configurable instance. */ public ObConfigurableInstance() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObConnectionFactory.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObConnectionFactory.java index b7ce04b2..78ca1b9f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObConnectionFactory.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObConnectionFactory.java @@ -69,7 +69,7 @@ private ObConnectionFactory(Codec codec, ChannelHandler handler, this.handler = handler; } - /** + /* * Init. */ @Override @@ -103,7 +103,7 @@ protected void initChannel(SocketChannel channel) { }); } - /** + /* * Create connection. */ @Override @@ -111,7 +111,7 @@ public Connection createConnection(Url url) throws Exception { throw new IllegalArgumentException("not support yet"); } - /** + /* * @see com.alipay.remoting.connection.ConnectionFactory#createConnection(java.lang.String, int, int) */ @Override @@ -125,7 +125,7 @@ public Connection createConnection(String targetIP, int targetPort, int connectT return conn; } - /** + /* * Create connection. */ @Override @@ -170,7 +170,7 @@ protected Channel doCreateConnection(String targetIP, int targetPort, int connec return future.channel(); } - /** + /* * init netty write buffer water mark */ private void initWriteBufferWaterMark() { @@ -191,7 +191,7 @@ private void initWriteBufferWaterMark() { lowWaterMark, highWaterMark)); } - /** + /* * New builder. */ public static ObConnectionFactoryBuilder newBuilder() { @@ -206,7 +206,7 @@ public static class ObConnectionFactoryBuilder { userProcessors); private ConfigurableInstance configurableInstance = new ObConfigurableInstance(); - /** + /* * Register user processor. */ public ObConnectionFactoryBuilder registerUserProcessor(UserProcessor processor) { @@ -225,7 +225,7 @@ public ObConnectionFactoryBuilder registerUserProcessor(UserProcessor process return this; } - /** + /* * Config write buffer water mark. */ public ObConnectionFactoryBuilder configWriteBufferWaterMark(int low, int high) { @@ -233,7 +233,7 @@ public ObConnectionFactoryBuilder configWriteBufferWaterMark(int low, int high) return this; } - /** + /* * Build. */ public ObConnectionFactory build() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObPacketCodec.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObPacketCodec.java index fcb5e046..8a221c64 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObPacketCodec.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObPacketCodec.java @@ -25,7 +25,7 @@ import io.netty.channel.ChannelHandler; public class ObPacketCodec implements Codec { - /** + /* * New encoder. */ @Override @@ -34,7 +34,7 @@ public ChannelHandler newEncoder() { ProtocolCode.fromBytes(ObTableProtocol.MAGIC_HEADER_FLAG)); } - /** + /* * New decoder. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObPacketFactory.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObPacketFactory.java index d93c047d..6e92a414 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObPacketFactory.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObPacketFactory.java @@ -33,7 +33,14 @@ public class ObPacketFactory implements CommandFactory { - /** + boolean reRouting; + + // construct ObPacketFactory + public ObPacketFactory(boolean reRouting) { + this.reRouting = reRouting; + } + + /* * Create request command. */ @Override @@ -61,12 +68,17 @@ private byte[] encodePayload(ObPayload payload) { // 2. assemble rpc packet header ObRpcPacketHeader rpcHeaderPacket = new ObRpcPacketHeader(); rpcHeaderPacket.setPcode(payload.getPcode()); + // flag + if (reRouting) { + rpcHeaderPacket.enableRerouting(); + } // us rpcHeaderPacket.setTimeout(payload.getTimeout() * 1000); rpcHeaderPacket.setTenantId(payload.getTenantId()); + rpcHeaderPacket.setPrvTenantId(payload.getTenantId()); // trace - rpcHeaderPacket.setTraceId0(payload.getSequence()); - rpcHeaderPacket.setTraceId1(payload.getUniqueId()); + rpcHeaderPacket.setTraceId0(payload.getUniqueId()); + rpcHeaderPacket.setTraceId1(payload.getSequence()); if (payload instanceof ObTableStreamRequest) { rpcHeaderPacket.setSessionId(((ObTableStreamRequest) payload).getSessionId()); @@ -84,7 +96,7 @@ private byte[] encodePayload(ObPayload payload) { return rpcPacket.encode(); } - /** + /* * Create timeout response. */ @Override @@ -93,7 +105,7 @@ public ObTablePacket createTimeoutResponse(InetSocketAddress address) { "connection {" + address.toString() + "} timeout", null); } - /** + /* * Create send failed response. */ @Override @@ -102,7 +114,7 @@ public ObTablePacket createSendFailedResponse(InetSocketAddress address, Throwab "connection {" + address.toString() + "} send failed", throwable); } - /** + /* * TODO tell client this connection has been closed */ // TODO for server processor @@ -111,7 +123,7 @@ public ObTablePacket createExceptionResponse(int id, Throwable t, String errMsg) return null; } - /** + /* * Create exception response. */ @Override @@ -119,7 +131,7 @@ public ObTablePacket createExceptionResponse(int id, ResponseStatus status) { return null; } - /** + /* * Create exception response. */ @Override @@ -127,7 +139,7 @@ public ObTablePacket createExceptionResponse(int id, String errMsg) { return null; } - /** + /* * Create response. */ @Override @@ -135,7 +147,7 @@ public ObTablePacket createResponse(Object responseObject, RemotingCommand reque return null; } - /** + /* * Create exception response. */ @Override @@ -143,7 +155,7 @@ public ObTablePacket createExceptionResponse(int id, ResponseStatus status, Thro return null; } - /** + /* * Create connection closed response. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableConnection.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableConnection.java index 930f56dc..26b2496c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableConnection.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableConnection.java @@ -17,51 +17,75 @@ package com.alipay.oceanbase.rpc.bolt.transport; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.exception.*; +import com.alipay.oceanbase.rpc.location.LocationUtil; import com.alipay.oceanbase.rpc.protocol.payload.impl.login.ObTableLoginRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.login.ObTableLoginResult; import com.alipay.oceanbase.rpc.table.ObTable; -import com.alipay.oceanbase.rpc.util.ObBytesString; -import com.alipay.oceanbase.rpc.util.Security; -import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; -import com.alipay.oceanbase.rpc.util.TraceUtil; +import com.alipay.oceanbase.rpc.util.*; import com.alipay.remoting.Connection; import org.slf4j.Logger; -import java.util.UUID; +import java.net.ConnectException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.MONITOR; +import static com.alipay.oceanbase.rpc.util.TraceUtil.formatTraceMessage; + public class ObTableConnection { - private static final Logger LOGGER = TableClientLoggerFactory - .getLogger(ObTableConnection.class); + private static final Logger LOGGER = TableClientLoggerFactory + .getLogger(ObTableConnection.class); private ObBytesString credential; - private long tenantId = 1; //默认值切勿不要随意改动 + private long tenantId = 1; //默认值切勿不要随意改动 private Connection connection; private final ObTable obTable; - private long uniqueId; - private AtomicLong sequence; + private long uniqueId; // as trace0 in rpc header + private AtomicLong sequence; // as trace1 in rpc header + private AtomicBoolean isReConnecting = new AtomicBoolean(false); // indicate is re-connecting or not - /** + public static long ipToLong(String strIp) { + String[] ip = strIp.split("\\."); + return (Long.parseLong(ip[0]) << 24) + (Long.parseLong(ip[1]) << 16) + + (Long.parseLong(ip[2]) << 8) + (Long.parseLong(ip[3])); + } + + /* * Ob table connection. */ public ObTableConnection(ObTable obTable) { this.obTable = obTable; } - /** + /* * Init. */ + public void init() throws Exception { - uniqueId = UUID.randomUUID().getMostSignificantBits(); + // sequence is a monotone increasing long value inside each connection sequence = new AtomicLong(); connect(); + /* layout of uniqueId(64 bytes) + * ip_: 32 + * port_: 16; + * is_user_request_: 1; + * is_ipv6_:1; + * reserved_: 14; + */ + long ip = ipToLong(connection.getLocalIP()); + long port = (long) connection.getLocalPort() << 32; + long isUserRequest = (1l << (32 + 16)); + long reserved = 0; + uniqueId = ip | port | isUserRequest | reserved; } - private synchronized boolean connect() throws Exception { + private boolean connect() throws Exception { if (checkAvailable()) { // double check status available return false; } + final long start = System.currentTimeMillis(); Exception cause = null; int tries = 0; int maxTryTimes = obTable.getObTableConnectTryTimes(); @@ -76,8 +100,12 @@ private synchronized boolean connect() throws Exception { "connect failed at " + tries + " try " + TraceUtil.formatIpPort(obTable), e); } } + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MONITOR.info(logMessage(null, "CONNECT", endpoint, System.currentTimeMillis() - start)); if (tries >= maxTryTimes) { + LOGGER.warn("connect failed after max " + maxTryTimes + " tries " + + TraceUtil.formatIpPort(obTable)); throw new ObTableServerConnectException("connect failed after max " + maxTryTimes + " tries " + TraceUtil.formatIpPort(obTable), cause); @@ -93,7 +121,8 @@ private synchronized boolean connect() throws Exception { return true; } - private synchronized void login() throws Exception { + private void login() throws Exception { + final long start = System.currentTimeMillis(); ObTableLoginRequest request = new ObTableLoginRequest(); request.setTenantName(obTable.getTenantName()); request.setUserName(obTable.getUserName()); @@ -113,6 +142,13 @@ private synchronized void login() throws Exception { && result.getCredential().length() > 0) { credential = result.getCredential(); tenantId = result.getTenantId(); + // Set version if missing + if (ObGlobal.obVsnMajor() == 0 && !result.getServerVersion().isEmpty()) { + // version should be set before login when direct mode + LocationUtil.parseObVerionFromLogin(result.getServerVersion()); + LOGGER.info("The OB_VERSION parsed from login result is: {}", + ObGlobal.OB_VERSION); + } break; } } catch (Exception e) { @@ -127,14 +163,19 @@ private synchronized void login() throws Exception { } } + String endpoint = obTable.getIp() + ":" + obTable.getPort(); + MONITOR.info(logMessage(formatTraceMessage(request), "LOGIN", endpoint, + System.currentTimeMillis() - start)); if (tries >= maxTryTimes) { + LOGGER.warn("login failed after max " + maxTryTimes + " tries " + + TraceUtil.formatIpPort(obTable)); throw new ObTableServerConnectException("login failed after max " + maxTryTimes + " tries " + TraceUtil.formatIpPort(obTable), cause); } } - /** + /* * Close. */ public void close() { @@ -145,7 +186,7 @@ public void close() { } } - /** + /* * Check status. */ public void checkStatus() throws Exception { @@ -168,50 +209,95 @@ public void checkStatus() throws Exception { } } + public void reConnectAndLogin(String msg) throws ObTableException { + try { + // 1. check the connection is available, force to close it + if (checkAvailable()) { + LOGGER.warn("The connection would be closed and reconnected if: " + + connection.getUrl()); + close(); + } + // 2. reconnect + reconnect(msg); + } catch (ConnectException ex) { + // cannot connect to ob server, need refresh table location + throw new ObTableServerConnectException(ex); + } catch (ObTableServerConnectException ex) { + throw ex; + } catch (Exception ex) { + throw new ObTableConnectionStatusException("check status failed", ex); + } + } + + /** + * Reconnect current connection and login + * + * @param msg the reconnect reason + * @exception Exception if connect successfully or connection already reconnected by others + * throw exception if connect failed + * + */ private void reconnect(String msg) throws Exception { - if (connect()) { - LOGGER.warn("reconnect success. reconnect reason: [{}]", msg); + if (isReConnecting.compareAndSet(false, true)) { + try { + if (connect()) { + LOGGER.warn("reconnect success. reconnect reason: [{}]", msg); + } else { + LOGGER.info( + "connection maybe reconnect by other thread. reconnect reason: [{}]", msg); + } + } catch (Exception e) { + throw e; + } finally { + if (!isReConnecting.compareAndSet(true, false)) { + LOGGER + .error( + "failed to set connecting to false after connect finished, reconnect reason: [{}]", + msg); + } + } } else { - LOGGER.warn("connection maybe reconnect by other thread. reconnect reason: [{}]", msg); + LOGGER.warn("There is someone connecting, no need reconnect"); + throw new ObTableException("This connection is already Connecting"); } } - /** + /* * Get credential. */ public ObBytesString getCredential() { return credential; } - /** + /* * Set credential. */ public void setCredential(ObBytesString credential) { this.credential = credential; } - /** + /* * Get tenant id. */ public long getTenantId() { return tenantId; } - /** + /* * Set tenant id. */ public void setTenantId(long tenantId) { this.tenantId = tenantId; } - /** + /* * Get connection. */ public Connection getConnection() { return connection; } - /** + /* * Get ob table. */ public ObTable getObTable() { @@ -222,9 +308,14 @@ private boolean checkAvailable() { if (connection == null) { return false; } - if (connection.getChannel() == null || !connection.getChannel().isActive()) { + if (connection.getChannel() == null) { + return false; + } + + if (!connection.getChannel().isActive()) { return false; } + if (credential == null) { return false; } @@ -238,17 +329,32 @@ private void generatePassSecret(ObTableLoginRequest request) { request.setPassScramble(scramble); } - /** + /* * Get unique id. */ public long getUniqueId() { return uniqueId; } - /** + /* * Get next sequence. */ public long getNextSequence() { return sequence.incrementAndGet(); } + + private String logMessage(String traceId, String methodName, String endpoint, long executeTime) { + if (org.apache.commons.lang.StringUtils.isNotBlank(endpoint)) { + endpoint = endpoint.replaceAll(",", "#"); + } + + StringBuilder stringBuilder = new StringBuilder(); + if (traceId != null) { + stringBuilder.append(traceId).append(" - "); + } + stringBuilder.append(methodName).append(",").append(endpoint).append(",") + .append(executeTime); + return stringBuilder.toString(); + } + } diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTablePacketHandler.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTablePacketHandler.java index 6b63959e..db795507 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTablePacketHandler.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTablePacketHandler.java @@ -36,21 +36,21 @@ public class ObTablePacketHandler implements CommandHandler { private static final Logger logger = TableClientLoggerFactory .getLogger(ObTablePacketHandler.class); - /** + /* * Unlike {@link com.alipay.remoting.rpc.protocol.RpcCommandHandler}, ObTable client only decodes ObNetPacket, it only has 16 bytes, * and all content will decode it in biz thread by {@link ObTableRemoting}. * So this class has no ProcessorManager logic */ private final ObTablePacketProcessor processor; - /** + /* * Ob table packet handler. */ public ObTablePacketHandler() { this.processor = new ObTablePacketProcessor(); } - /** + /* * Handle command. */ @Override @@ -88,7 +88,7 @@ private void processExceptionForSingleCommand(Object msg, Throwable t) { logger.error(LCD.convert("01-00021"), id, t); } - /** + /* * Register processor. */ @Override @@ -96,7 +96,7 @@ public void registerProcessor(CommandCode cmd, RemotingProcessor processor) { throw new IllegalArgumentException("not support yet"); } - /** + /* * Register default executor. */ @Override @@ -104,7 +104,7 @@ public void registerDefaultExecutor(ExecutorService executor) { throw new IllegalArgumentException("not support yet"); } - /** + /* * Get default executor. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTablePacketProcessor.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTablePacketProcessor.java index 2d66277e..da7e329e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTablePacketProcessor.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTablePacketProcessor.java @@ -35,7 +35,7 @@ public class ObTablePacketProcessor implements RemotingProcessor private static final Logger logger = TableClientLoggerFactory .getLogger(ObTablePacketProcessor.class); - /** + /* * Process. */ @Override @@ -62,6 +62,7 @@ public void process(RemotingContext ctx, ObTablePacket msg, ExecutorService defa .warn("Cannot find InvokeFuture, maybe already timeout, id={}, from={} ", msg.getId(), RemotingUtil.parseRemoteAddress(ctx.getChannelContext().channel())); + msg.releaseByteBuf(); } } finally { if (null != oldClassLoader) { @@ -71,7 +72,7 @@ public void process(RemotingContext ctx, ObTablePacket msg, ExecutorService defa } - /** + /* * Get executor. */ @Override @@ -79,7 +80,7 @@ public ExecutorService getExecutor() { return null; } - /** + /* * Set executor. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java index 8dbebb6b..9171f52e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java +++ b/src/main/java/com/alipay/oceanbase/rpc/bolt/transport/ObTableRemoting.java @@ -19,13 +19,10 @@ import com.alipay.oceanbase.rpc.bolt.protocol.ObTablePacket; import com.alipay.oceanbase.rpc.bolt.protocol.ObTablePacketCode; -import com.alipay.oceanbase.rpc.exception.ExceptionUtil; -import com.alipay.oceanbase.rpc.exception.ObTableRoutingWrongException; -import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; -import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; -import com.alipay.oceanbase.rpc.protocol.payload.Credentialable; -import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; -import com.alipay.oceanbase.rpc.protocol.payload.ObRpcResultCode; +import com.alipay.oceanbase.rpc.exception.*; +import com.alipay.oceanbase.rpc.protocol.packet.ObCompressType; +import com.alipay.oceanbase.rpc.protocol.payload.*; +import com.alipay.oceanbase.rpc.protocol.payload.impl.login.ObTableLoginRequest; import com.alipay.oceanbase.rpc.util.ObPureCrc32C; import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; import com.alipay.oceanbase.rpc.util.TraceUtil; @@ -34,18 +31,21 @@ import io.netty.buffer.ByteBuf; import org.slf4j.Logger; +import static com.alipay.oceanbase.rpc.protocol.packet.ObCompressType.INVALID_COMPRESSOR; +import static com.alipay.oceanbase.rpc.protocol.packet.ObCompressType.NONE_COMPRESSOR; + public class ObTableRemoting extends BaseRemoting { private static final Logger logger = TableClientLoggerFactory.getLogger(ObTableRemoting.class); - /** + /* * Ob table remoting. */ public ObTableRemoting(CommandFactory commandFactory) { super(commandFactory); } - /** + /* * Invoke sync. */ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload request, @@ -56,10 +56,18 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques request.setUniqueId(conn.getUniqueId()); if (request instanceof Credentialable) { + if (conn.getCredential() == null) { + String errMessage = TraceUtil.formatTraceMessage(conn, request, + "credential is null"); + logger.warn(errMessage); + throw new ObTableUnexpectedException(errMessage); + } ((Credentialable) request).setCredential(conn.getCredential()); } - - if (request instanceof AbstractPayload) { + if (request instanceof ObTableLoginRequest) { + // setting sys tenant in rpc header when login + ((ObTableLoginRequest) request).setTenantId(1); + } else if (request instanceof AbstractPayload) { ((AbstractPayload) request).setTenantId(conn.getTenantId()); } @@ -78,6 +86,7 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques String errMessage = TraceUtil.formatTraceMessage(conn, request, "get an error response: " + response.getMessage()); logger.warn(errMessage); + response.releaseByteBuf(); ExceptionUtil.throwObTableTransportException(errMessage, response.getTransportCode()); return null; } @@ -85,17 +94,17 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques try { // decode packet header first response.decodePacketHeader(); - - ByteBuf buf = response.getPacketContentBuf(); - - // If response indicates the request is routed to wrong server, we should refresh the routing meta. - if (response.getHeader().isRoutingWrong()) { - String errMessage = TraceUtil.formatTraceMessage(conn, request, - "routed to the wrong server: " + response.getMessage()); + ObCompressType compressType = response.getHeader().getObCompressType(); + if (compressType != INVALID_COMPRESSOR && compressType != NONE_COMPRESSOR) { + String errMessage = TraceUtil.formatTraceMessage( + conn, + request, + "Rpc Result is compressed. Java Client is not supported. msg:" + + response.getMessage()); logger.warn(errMessage); - throw new ObTableRoutingWrongException(errMessage); + throw new FeatureNotSupportedException(errMessage); } - + ByteBuf buf = response.getPacketContentBuf(); // verify checksum long expected_checksum = response.getHeader().getChecksum(); byte[] content = new byte[buf.readableBytes()]; @@ -112,11 +121,28 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques // decode ResultCode for response packet ObRpcResultCode resultCode = new ObRpcResultCode(); resultCode.decode(buf); - + // If response indicates the request is routed to wrong server, we should refresh the routing meta. + if (response.getHeader().isRoutingWrong()) { + String errMessage = TraceUtil.formatTraceMessage(conn, request, + "routed to the wrong server: " + response.getMessage()); + logger.warn(errMessage); + if (needFetchAll(resultCode.getRcode(), resultCode.getPcode())) { + throw new ObTableNeedFetchAllException(errMessage); + } else if (needFetchPartial(resultCode.getRcode())) { + throw new ObTableRoutingWrongException(errMessage); + } else { + // Encountered an unexpected RoutingWrong error code, + // possibly due to the client error code version being behind the observer's version. + // Attempting a full refresh here + // and delegating to the upper-level call to determine whether to throw the exception to the user based on the retry result. + logger.warn("get unexpected error code: {}", response.getMessage()); + throw new ObTableNeedFetchAllException(errMessage); + } + } if (resultCode.getRcode() != 0) { ExceptionUtil.throwObTableException(conn.getObTable().getIp(), conn.getObTable() - .getPort(), response.getHeader().getTraceId0(), response.getHeader() - .getTraceId1(), resultCode.getRcode()); + .getPort(), response.getHeader().getTraceId1(), response.getHeader() + .getTraceId0(), resultCode.getRcode(), resultCode.getErrMsg()); return null; } @@ -125,8 +151,8 @@ public ObPayload invokeSync(final ObTableConnection conn, final ObPayload reques if (response.getCmdCode() instanceof ObTablePacketCode) { payload = ((ObTablePacketCode) response.getCmdCode()).newPayload(response .getHeader()); - payload.setSequence(response.getHeader().getTraceId0()); - payload.setUniqueId(response.getHeader().getTraceId1()); + payload.setSequence(response.getHeader().getTraceId1()); + payload.setUniqueId(response.getHeader().getTraceId0()); } else { String errMessage = TraceUtil.formatTraceMessage(conn, response, "receive unexpected command code: " + response.getCmdCode().value()); @@ -153,4 +179,29 @@ protected InvokeFuture createInvokeFuture(Connection conn, RemotingCommand reque return new ObClientFuture(request.getId()); } + // schema changed + private boolean needFetchAll(int errorCode, int pcode) { + return errorCode == ResultCodes.OB_SCHEMA_ERROR.errorCode + || errorCode == ResultCodes.OB_TABLE_NOT_EXIST.errorCode + || errorCode == ResultCodes.OB_TABLET_NOT_EXIST.errorCode + || errorCode == ResultCodes.OB_LS_NOT_EXIST.errorCode + || (pcode == Pcodes.OB_TABLE_API_LS_EXECUTE && errorCode == ResultCodes.OB_NOT_MASTER.errorCode); + } + private boolean needFetchPartial(int errorCode) { + return errorCode == ResultCodes.OB_LOCATION_LEADER_NOT_EXIST.errorCode + || errorCode == ResultCodes.OB_NOT_MASTER.errorCode + || errorCode == ResultCodes.OB_RS_NOT_MASTER.errorCode + || errorCode == ResultCodes.OB_RS_SHUTDOWN.errorCode + || errorCode == ResultCodes.OB_RPC_SEND_ERROR.errorCode + || errorCode == ResultCodes.OB_RPC_POST_ERROR.errorCode + || errorCode == ResultCodes.OB_PARTITION_NOT_EXIST.errorCode + || errorCode == ResultCodes.OB_LOCATION_NOT_EXIST.errorCode + || errorCode == ResultCodes.OB_PARTITION_IS_STOPPED.errorCode + || errorCode == ResultCodes.OB_PARTITION_IS_BLOCKED.errorCode + || errorCode == ResultCodes.OB_SERVER_IS_INIT.errorCode + || errorCode == ResultCodes.OB_SERVER_IS_STOPPING.errorCode + || errorCode == ResultCodes.OB_TENANT_NOT_IN_SERVER.errorCode + || errorCode == ResultCodes.OB_TRANS_RPC_TIMEOUT.errorCode + || errorCode == ResultCodes.OB_NO_READABLE_REPLICA.errorCode; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/checkandmutate/CheckAndInsUp.java b/src/main/java/com/alipay/oceanbase/rpc/checkandmutate/CheckAndInsUp.java new file mode 100644 index 00000000..e588100c --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/checkandmutate/CheckAndInsUp.java @@ -0,0 +1,101 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.checkandmutate; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; +import com.alipay.oceanbase.rpc.mutation.InsertOrUpdate; +import com.alipay.oceanbase.rpc.mutation.Row; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; +import com.alipay.oceanbase.rpc.table.api.Table; +import com.alipay.oceanbase.rpc.table.api.TableQuery; + +import java.util.ArrayList; +import java.util.List; + +public class CheckAndInsUp { + private Table client; + private String tableName; + private ObTableFilter filter; + private InsertOrUpdate insUp; + boolean checkExists = true; + + public CheckAndInsUp(ObTableFilter filter, InsertOrUpdate insUp, boolean check_exists) + throws IllegalArgumentException { + this.filter = filter; + this.insUp = insUp; + this.checkExists = check_exists; + this.tableName = null; + this.client = null; + } + + public CheckAndInsUp(Table client, String tableName, ObTableFilter filter, + InsertOrUpdate insUp, boolean check_exists) + throws IllegalArgumentException { + this.client = client; + this.tableName = tableName; + this.filter = filter; + this.insUp = insUp; + this.checkExists = check_exists; + } + + public Row getRowKey() { + return insUp.getRowKey(); + } + + public ObTableFilter getFilter() { + return filter; + } + + public InsertOrUpdate getInsUp() { + return insUp; + } + + public boolean isCheckExists() { + return checkExists; + } + + public MutationResult execute() throws Exception { + if (null == tableName || tableName.isEmpty()) { + throw new ObTableException("table name is null"); + } else if (null == client) { + throw new ObTableException("client is null"); + } else if (!(client instanceof ObTableClient)) { + throw new ObTableException("the client must be table clinet"); + } + + TableQuery query = client.query(tableName); + query.setFilter(filter); + Row rowKey = getRowKey(); + List ranges = new ArrayList<>(); + ObNewRange range = new ObNewRange(); + range.setStartKey(ObRowKey.getInstance(insUp.getRowKey().getValues())); + range.setEndKey(ObRowKey.getInstance(insUp.getRowKey().getValues())); + ranges.add(range); + query.getObTableQuery().setKeyRanges(ranges); + ObTableOperation operation = ObTableOperation.getInstance(ObTableOperationType.INSERT_OR_UPDATE, + insUp.getRowKey().getValues(), insUp.getColumns(), insUp.getValues()); + + return new MutationResult(((ObTableClient)client).mutationWithFilter(query, rowKey, ranges, operation, false, true, checkExists)); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/constant/Constants.java b/src/main/java/com/alipay/oceanbase/rpc/constant/Constants.java index 397fa814..74019161 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/constant/Constants.java +++ b/src/main/java/com/alipay/oceanbase/rpc/constant/Constants.java @@ -32,4 +32,5 @@ public interface Constants { String OB_ROUTE_POLICY = "ob_route_policy"; String OCP_ROOT_SERVICE_ACTION = "ObRootServiceInfo"; String OCP_IDC_REGION_ACTION = "ObIDCRegionInfo"; + Long INVALID_TABLET_ID = 0L; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ExceptionUtil.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ExceptionUtil.java index c76aead9..22900bdf 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ExceptionUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ExceptionUtil.java @@ -20,130 +20,76 @@ import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; -import static com.alipay.oceanbase.rpc.protocol.payload.ResultCodes.OB_DESERIALIZE_ERROR; +import java.util.Objects; + +import static com.alipay.oceanbase.rpc.protocol.payload.ResultCodes.*; public class ExceptionUtil { - /** + /* * throw ObTableException based on error code * TODO 错误信息这里应该要加上表名/字段等，但是报错的地方没有表名上下文，比较麻烦 * * @param errorCode error code return from Ob Server */ public static void throwObTableException(int errorCode) { - ResultCodes resultCodes = ResultCodes.valueOf(errorCode); - if (resultCodes != ResultCodes.OB_SUCCESS) { - throw convertToObTableException("", 0, 0, 0, resultCodes); + if (errorCode != ResultCodes.OB_SUCCESS.errorCode) { + throw convertToObTableException("", 0, 0, 0, errorCode, ""); } } - /** + /* * throw ObTableException based on error code * TODO 错误信息这里应该要加上表名/字段等，但是报错的地方没有表名上下文，比较麻烦 * * @param errorCode error code return from Ob Server */ public static void throwObTableException(String host, int port, long sequence, long uniqueId, - int errorCode) { - ResultCodes resultCodes = ResultCodes.valueOf(errorCode); - if (resultCodes != ResultCodes.OB_SUCCESS) { - throw convertToObTableException(host, port, sequence, uniqueId, resultCodes); + int errorCode, String errorMessage) { + if (errorCode != ResultCodes.OB_SUCCESS.errorCode) { + throw convertToObTableException(host, port, sequence, uniqueId, errorCode, errorMessage); } } - /** + /* * throw ObTableException based on error code * TODO 错误信息这里应该要加上表名/字段等，但是报错的地方没有表名上下文，比较麻烦 * * @param resultCodes error code return from Ob Server */ public static ObTableException convertToObTableException(String host, int port, long sequence, - long uniqueId, ResultCodes resultCodes) { - String trace = String.format("Y%X-%016X", sequence, uniqueId); + long uniqueId, int errorCode, + String errorMessage) { + String trace = String.format("Y%X-%016X", uniqueId, sequence); String server = host + ":" + port; - switch (resultCodes) { - case OB_ERR_PRIMARY_KEY_DUPLICATE: - return new ObTableDuplicateKeyException("[" + trace + "] [" + resultCodes.name() - + "] server [" + server + "] ", - resultCodes.errorCode); - case OB_ERR_UNKNOWN_TABLE: - return new ObTableNotExistException("[" + trace - + "] [OB_ERR_UNKNOWN_TABLE] server [" + server - + "]table not exist", resultCodes.errorCode); - case OB_ERR_COLUMN_NOT_FOUND: - return new ObTableException("[" + trace + "] [OB_ERR_COLUMN_NOT_FOUND] server [" - + server + "]column not found", resultCodes.errorCode); - case OB_OBJ_TYPE_ERROR: - return new ObTableException("[" + trace + "] [OB_OBJ_TYPE_ERROR] server [" + server - + "]rowkey/column type not match", - resultCodes.errorCode); - case OB_BAD_NULL_ERROR: - return new ObTableException("[" + trace + "] [OB_BAD_NULL_ERROR] server [" + server - + "]column doesn't have a default value", - resultCodes.errorCode); - case OB_INVALID_ARGUMENT: - return new ObTableException("[" + trace + "] [OB_INVALID_ARGUMENT] server [" - + server + "]ob table arguments invalid", - resultCodes.errorCode); - case OB_DESERIALIZE_ERROR: // 用户名错误是这个异常码 - return new ObTableAuthException("[" + trace + "] [OB_DESERIALIZE_ERROR] server [" - + server + "]maybe invalid username, errorCode: " - + OB_DESERIALIZE_ERROR, resultCodes.errorCode); - case OB_PASSWORD_WRONG: - return new ObTableAuthException("[" + trace + "] [OB_PASSWORD_WRONG] server [" - + server + "]access deny", resultCodes.errorCode); - case OB_LOCATION_LEADER_NOT_EXIST: - case OB_NOT_MASTER: - case OB_RS_NOT_MASTER: - case OB_RS_SHUTDOWN: - return new ObTableMasterChangeException("[" + trace + "] [" + resultCodes.name() - + "] server [" + server + "]", - resultCodes.errorCode); - case OB_RPC_CONNECT_ERROR: - return new ObTableServerDownException("[" + trace + "] [" + resultCodes.name() - + "] server [" + server + "]", - resultCodes.errorCode); - case OB_PARTITION_NOT_EXIST: - case OB_PARTITION_IS_STOPPED: - case OB_LOCATION_NOT_EXIST: - return new ObTablePartitionChangeException("[" + trace + "] [" + resultCodes.name() - + "] server [" + server + "]", - resultCodes.errorCode); - case OB_SERVER_IS_INIT: - case OB_SERVER_IS_STOPPING: - return new ObTableServerStatusChangeException("[" + trace + "] [" - + resultCodes.name() + "] server [" - + server + "]", resultCodes.errorCode); - case OB_TENANT_NOT_IN_SERVER: - return new ObTableUnitMigrateException("[" + trace + "] [" + resultCodes.name() - + "] server [" + server + "]", - resultCodes.errorCode); - case OB_TRANS_RPC_TIMEOUT: - return new ObTableTransactionRpcTimeout("[" + trace + "] [" + resultCodes.name() - + "] server [" + server + "]", - resultCodes.errorCode); - case OB_NO_READABLE_REPLICA: - return new ObTableNoReadableReplicaException("[" + trace + "] [" - + resultCodes.name() + "] server [" - + server + "]", resultCodes.errorCode); - case OB_REPLICA_NOT_READABLE: - return new ObTableReplicaNotReadableException("[" + trace + "] [" - + resultCodes.name() + "] server [" - + server + "]", resultCodes.errorCode); - case OB_TIMEOUT: - case OB_TRANS_TIMEOUT: - case OB_WAITQUEUE_TIMEOUT: - return new ObTableServerTimeoutException("[" + trace + "] [" + resultCodes.name() - + "] server [" + server + "]", - resultCodes.errorCode); - default: - return new ObTableUnexpectedException("[" + trace + "] [" + resultCodes.name() - + "] server [" + server + "]", - resultCodes.errorCode); + String errMsg = Objects.equals(errorMessage, "") ? "error occur in server" : errorMessage; + ResultCodes resultCodes = ResultCodes.valueOf(errorCode); + if (resultCodes == null) { + return new ObTableUnexpectedException("[" + trace + "] [" + "unknown errcode: " + + errorCode + "] server [" + server + "]", + errorCode); + } + + if (resultCodes.errorCode == OB_ERR_KV_GLOBAL_INDEX_ROUTE.errorCode) { + return new ObTableGlobalIndexRouteException("[" + String.valueOf(resultCodes.errorCode) + + "]" + "[" + resultCodes.name() + "]" + + "[" + errMsg + "]" + "[" + server + "]" + + "[" + trace + "]", resultCodes.errorCode); + } else if (resultCodes.errorCode == OB_TENANT_NOT_IN_SERVER.errorCode) { + return new ObTableTenantNotInServerException("[" + + String.valueOf(resultCodes.errorCode) + + "]" + "[" + resultCodes.name() + "]" + + "[" + errMsg + "]" + "[" + server + "]" + + "[" + trace + "]", resultCodes.errorCode); + } else { + // [errCode][errCodeName][errMsg][server][trace] + return new ObTableException("[" + String.valueOf(resultCodes.errorCode) + "]" + "[" + + resultCodes.name() + "]" + "[" + errMsg + "]" + "[" + + server + "]" + "[" + trace + "]", resultCodes.errorCode); } } - /** + /* * throw ObTableTransportException based on transport error code * TODO 错误信息这里应该要加上表名/字段等，但是报错的地方没有表名上下文，比较麻烦 * diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/FeatureNotSupportedException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/FeatureNotSupportedException.java index 1c83b759..4596e454 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/FeatureNotSupportedException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/FeatureNotSupportedException.java @@ -21,20 +21,20 @@ public class FeatureNotSupportedException extends RuntimeException { private int errorCode; - /** + /* * Feature not supported exception. */ public FeatureNotSupportedException() { } - /** + /* * Feature not supported exception. */ public FeatureNotSupportedException(int errorCode) { this("error code: " + errorCode, errorCode); } - /** + /* * Feature not supported exception. */ public FeatureNotSupportedException(String message, int errorCode) { @@ -42,28 +42,28 @@ public FeatureNotSupportedException(String message, int errorCode) { this.errorCode = errorCode; } - /** + /* * Feature not supported exception. */ public FeatureNotSupportedException(String message) { super(message); } - /** + /* * Feature not supported exception. */ public FeatureNotSupportedException(String message, Throwable cause) { super(message, cause); } - /** + /* * Feature not supported exception. */ public FeatureNotSupportedException(Throwable cause) { super(cause); } - /** + /* * Get error code. */ public int getErrorCode() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/GenerateColumnParseException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/GenerateColumnParseException.java index 36639db0..2751b2a3 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/GenerateColumnParseException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/GenerateColumnParseException.java @@ -21,20 +21,20 @@ public class GenerateColumnParseException extends RuntimeException { private int errorCode; - /** + /* * Generate column parse exception. */ public GenerateColumnParseException() { } - /** + /* * Generate column parse exception. */ public GenerateColumnParseException(int errorCode) { this("error code: " + errorCode, errorCode); } - /** + /* * Generate column parse exception. */ public GenerateColumnParseException(String message, int errorCode) { @@ -42,28 +42,28 @@ public GenerateColumnParseException(String message, int errorCode) { this.errorCode = errorCode; } - /** + /* * Generate column parse exception. */ public GenerateColumnParseException(String message) { super(message); } - /** + /* * Generate column parse exception. */ public GenerateColumnParseException(String message, Throwable cause) { super(message, cause); } - /** + /* * Generate column parse exception. */ public GenerateColumnParseException(Throwable cause) { super(cause); } - /** + /* * Get error code. */ public int getErrorCode() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObShardTableException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObShardTableException.java index f06c86c4..cdb8ef5f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObShardTableException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObShardTableException.java @@ -21,20 +21,20 @@ public class ObShardTableException extends RuntimeException { private int errorCode; - /** + /* * Ob shard table exception. */ public ObShardTableException() { } - /** + /* * Ob shard table exception. */ public ObShardTableException(int errorCode) { this("error code: " + errorCode, errorCode); } - /** + /* * Ob shard table exception. */ public ObShardTableException(String message, int errorCode) { @@ -42,28 +42,28 @@ public ObShardTableException(String message, int errorCode) { this.errorCode = errorCode; } - /** + /* * Ob shard table exception. */ public ObShardTableException(String message) { super(message); } - /** + /* * Ob shard table exception. */ public ObShardTableException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob shard table exception. */ public ObShardTableException(Throwable cause) { super(cause); } - /** + /* * Get error code. */ public int getErrorCode() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableAuthException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableAuthException.java index 3b9cf744..5863c6ff 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableAuthException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableAuthException.java @@ -19,41 +19,41 @@ public class ObTableAuthException extends ObTableException { - /** + /* * Ob table auth exception. */ public ObTableAuthException() { } - /** + /* * Ob table auth exception. */ public ObTableAuthException(int errorCode) { super(errorCode); } - /** + /* * Ob table auth exception. */ public ObTableAuthException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table auth exception. */ public ObTableAuthException(String message) { super(message); } - /** + /* * Ob table auth exception. */ public ObTableAuthException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table auth exception. */ public ObTableAuthException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableCloseException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableCloseException.java index d369c2cb..662e727f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableCloseException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableCloseException.java @@ -19,41 +19,41 @@ public class ObTableCloseException extends ObTableException { - /** + /* * Ob table close exception. */ public ObTableCloseException() { } - /** + /* * Ob table close exception. */ public ObTableCloseException(int errorCode) { super(errorCode); } - /** + /* * Ob table close exception. */ public ObTableCloseException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table close exception. */ public ObTableCloseException(String message) { super(message); } - /** + /* * Ob table close exception. */ public ObTableCloseException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table close exception. */ public ObTableCloseException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableConnectionStatusException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableConnectionStatusException.java index 9a8ac021..dda778db 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableConnectionStatusException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableConnectionStatusException.java @@ -18,41 +18,41 @@ package com.alipay.oceanbase.rpc.exception; public class ObTableConnectionStatusException extends ObTableException { - /** + /* * Ob table connection status exception. */ public ObTableConnectionStatusException() { } - /** + /* * Ob table connection status exception with error code. */ public ObTableConnectionStatusException(int errorCode) { super(errorCode); } - /** + /* * Ob table connection status exception with message and error code. */ public ObTableConnectionStatusException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table connection status exception with message. */ public ObTableConnectionStatusException(String message) { super(message); } - /** + /* * Ob table connection status exception with message and cause. */ public ObTableConnectionStatusException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table connection status exception with cause. */ public ObTableConnectionStatusException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableConnectionUnWritableException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableConnectionUnWritableException.java index e36b714f..9c50fcff 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableConnectionUnWritableException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableConnectionUnWritableException.java @@ -18,41 +18,41 @@ package com.alipay.oceanbase.rpc.exception; public class ObTableConnectionUnWritableException extends ObTableException { - /** + /* * Ob table connection un-writable exception. */ public ObTableConnectionUnWritableException() { } - /** + /* * Ob table connection un-writable exception with error code */ public ObTableConnectionUnWritableException(int errorCode) { super(errorCode); } - /** + /* * Ob table connection un-writable exception with message and error code. */ public ObTableConnectionUnWritableException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table connection un-writable exception with message. */ public ObTableConnectionUnWritableException(String message) { super(message); } - /** + /* * Ob table connection un-writable exception with message and cause. */ public ObTableConnectionUnWritableException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table connection un-writable exception with cause. */ public ObTableConnectionUnWritableException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableDuplicateKeyException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableDuplicateKeyException.java index cd44fc4e..d2eef8df 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableDuplicateKeyException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableDuplicateKeyException.java @@ -19,42 +19,42 @@ public class ObTableDuplicateKeyException extends ObTableException { - /** + /* * Ob table duplicate key exception. */ public ObTableDuplicateKeyException() { } - /** + /* * Ob table duplicate key exception. */ public ObTableDuplicateKeyException(int errorCode) { super(errorCode); } - /** + /* * Ob table duplicate key exception. */ public ObTableDuplicateKeyException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table duplicate key exception. */ public ObTableDuplicateKeyException(String message) { super(message); } - /** + /* * Ob table duplicate key exception. */ public ObTableDuplicateKeyException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table duplicate key exception. */ public ObTableDuplicateKeyException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableEntryRefreshException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableEntryRefreshException.java index a4cdac39..f7a1af2f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableEntryRefreshException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableEntryRefreshException.java @@ -19,48 +19,48 @@ public class ObTableEntryRefreshException extends ObTableException { - /** + /* * Ob table entry refresh exception. */ public ObTableEntryRefreshException() { } - /** + /* * Ob table entry refresh exception. */ public ObTableEntryRefreshException(int errorCode) { super(errorCode); } - /** + /* * Ob table entry refresh exception. */ public ObTableEntryRefreshException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table entry refresh exception. */ public ObTableEntryRefreshException(String message) { super(message); } - /** + /* * Ob table entry refresh exception. */ public ObTableEntryRefreshException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table entry refresh exception. */ public ObTableEntryRefreshException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableException.java index 7b5c290c..230cbfb4 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableException.java @@ -22,20 +22,20 @@ public class ObTableException extends RuntimeException { private int errorCode; private boolean needRefreshTableEntry; - /** + /* * Ob table exception. */ public ObTableException() { } - /** + /* * Ob table exception. */ public ObTableException(int errorCode) { this("error code: " + errorCode, errorCode); } - /** + /* * Ob table exception. */ public ObTableException(String message, int errorCode) { @@ -43,14 +43,14 @@ public ObTableException(String message, int errorCode) { this.errorCode = errorCode; } - /** + /* * Ob table exception. */ public ObTableException(String message) { super(message); } - /** + /* * Ob table exception. */ public ObTableException(String message, Throwable cause) { @@ -61,21 +61,21 @@ public ObTableException(String message, Throwable cause) { } } - /** + /* * Ob table exception. */ public ObTableException(Throwable cause) { super(cause); } - /** + /* * Get error code. */ public int getErrorCode() { return errorCode; } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableGetException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableGetException.java index ca34448b..7f0a9620 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableGetException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableGetException.java @@ -19,41 +19,41 @@ public class ObTableGetException extends ObTableException { - /** + /* * Ob table get exception. */ public ObTableGetException() { } - /** + /* * Ob table get exception. */ public ObTableGetException(int errorCode) { super(errorCode); } - /** + /* * Ob table get exception. */ public ObTableGetException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table get exception. */ public ObTableGetException(String message) { super(message); } - /** + /* * Ob table get exception. */ public ObTableGetException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table get exception. */ public ObTableGetException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableGlobalIndexRouteException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableGlobalIndexRouteException.java new file mode 100644 index 00000000..6545c17b --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableGlobalIndexRouteException.java @@ -0,0 +1,69 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.exception; + +public class ObTableGlobalIndexRouteException extends ObTableException { + + /* + * Ob table global index route exception. + */ + public ObTableGlobalIndexRouteException() { + } + + /* + * Ob table global index route exception. + */ + public ObTableGlobalIndexRouteException(int errorCode) { + super(errorCode); + } + + /* + * Ob table global index route exception. + */ + public ObTableGlobalIndexRouteException(String message, int errorCode) { + super(message, errorCode); + } + + /* + * Ob table global index route exception. + */ + public ObTableGlobalIndexRouteException(String message) { + super(message); + } + + /* + * Ob table global index route exception. + */ + public ObTableGlobalIndexRouteException(String message, Throwable cause) { + super(message, cause); + } + + /* + * Ob table global index route exception. + */ + public ObTableGlobalIndexRouteException(Throwable cause) { + super(cause); + } + + /* + * Is need refresh table entry. + */ + public boolean isNeedRefreshTableEntry() { + return false; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableLoginException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableLoginException.java index a4005f66..092afa7c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableLoginException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableLoginException.java @@ -19,41 +19,41 @@ public class ObTableLoginException extends ObTableException { - /** + /* * Ob table login exception. */ public ObTableLoginException() { } - /** + /* * Ob table login exception. */ public ObTableLoginException(int errorCode) { super(errorCode); } - /** + /* * Ob table login exception. */ public ObTableLoginException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table login exception. */ public ObTableLoginException(String message) { super(message); } - /** + /* * Ob table login exception. */ public ObTableLoginException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table login exception. */ public ObTableLoginException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableMasterChangeException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableMasterChangeException.java index 4e0d43c2..d03f8332 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableMasterChangeException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableMasterChangeException.java @@ -19,48 +19,48 @@ public class ObTableMasterChangeException extends ObTableException { - /** + /* * Ob table master change exception. */ public ObTableMasterChangeException() { } - /** + /* * Ob table master change exception. */ public ObTableMasterChangeException(int errorCode) { super(errorCode); } - /** + /* * Ob table master change exception. */ public ObTableMasterChangeException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table master change exception. */ public ObTableMasterChangeException(String message) { super(message); } - /** + /* * Ob table master change exception. */ public ObTableMasterChangeException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table master change exception. */ public ObTableMasterChangeException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNeedFetchAllException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNeedFetchAllException.java new file mode 100644 index 00000000..3e5a7715 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNeedFetchAllException.java @@ -0,0 +1,68 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.exception; + +public class ObTableNeedFetchAllException extends ObTableException { + /* + * Ob table routing wrong exception. + */ + public ObTableNeedFetchAllException() { + } + + /* + * Ob table routing wrong exception with error code. + */ + public ObTableNeedFetchAllException(int errorCode) { + super(errorCode); + } + + /* + * Ob table routing wrong exception with message and error code. + */ + public ObTableNeedFetchAllException(String message, int errorCode) { + super(message, errorCode); + } + + /* + * Ob table routing wrong exception with message. + */ + public ObTableNeedFetchAllException(String message) { + super(message); + } + + /* + * Ob table routing wrong exception with message and cause. + */ + public ObTableNeedFetchAllException(String message, Throwable cause) { + super(message, cause); + } + + /* + * Ob table routing wrong exception with cause. + */ + public ObTableNeedFetchAllException(Throwable cause) { + super(cause); + } + + /* + * Is need refresh table entry. + */ + public boolean isNeedRefreshTableEntry() { + return true; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNoMasterException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNoMasterException.java index c507f766..4d5d395b 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNoMasterException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNoMasterException.java @@ -19,41 +19,41 @@ public class ObTableNoMasterException extends ObTableException { - /** + /* * Ob table no master exception. */ public ObTableNoMasterException() { } - /** + /* * Ob table no master exception. */ public ObTableNoMasterException(int errorCode) { super(errorCode); } - /** + /* * Ob table no master exception. */ public ObTableNoMasterException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table no master exception. */ public ObTableNoMasterException(String message) { super(message); } - /** + /* * Ob table no master exception. */ public ObTableNoMasterException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table no master exception. */ public ObTableNoMasterException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNoReadableReplicaException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNoReadableReplicaException.java index ef07074f..aa0c5412 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNoReadableReplicaException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNoReadableReplicaException.java @@ -19,48 +19,48 @@ public class ObTableNoReadableReplicaException extends ObTableException { - /** + /* * Ob table no readable replica exception. */ public ObTableNoReadableReplicaException() { } - /** + /* * Ob table no readable replica exception. */ public ObTableNoReadableReplicaException(int errorCode) { super(errorCode); } - /** + /* * Ob table no readable replica exception. */ public ObTableNoReadableReplicaException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table no readable replica exception. */ public ObTableNoReadableReplicaException(String message) { super(message); } - /** + /* * Ob table no readable replica exception. */ public ObTableNoReadableReplicaException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table no readable replica exception. */ public ObTableNoReadableReplicaException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNotExistException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNotExistException.java index 4a269b1e..61ab09eb 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNotExistException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableNotExistException.java @@ -19,41 +19,41 @@ public class ObTableNotExistException extends ObTableEntryRefreshException { - /** + /* * Ob table not exist exception. */ public ObTableNotExistException() { } - /** + /* * Ob table not exist exception. */ public ObTableNotExistException(int errorCode) { super(errorCode); } - /** + /* * Ob table not exist exception. */ public ObTableNotExistException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table not exist exception. */ public ObTableNotExistException(String message) { super(message); } - /** + /* * Ob table not exist exception. */ public ObTableNotExistException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table not exist exception. */ public ObTableNotExistException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionChangeException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionChangeException.java index cc203ca1..0a2fc442 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionChangeException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionChangeException.java @@ -19,48 +19,48 @@ public class ObTablePartitionChangeException extends ObTableException { - /** + /* * Ob table partition change exception. */ public ObTablePartitionChangeException() { } - /** + /* * Ob table partition change exception. */ public ObTablePartitionChangeException(int errorCode) { super(errorCode); } - /** + /* * Ob table partition change exception. */ public ObTablePartitionChangeException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table partition change exception. */ public ObTablePartitionChangeException(String message) { super(message); } - /** + /* * Ob table partition change exception. */ public ObTablePartitionChangeException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table partition change exception. */ public ObTablePartitionChangeException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionConsistentException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionConsistentException.java index 33bb0860..03631ad7 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionConsistentException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionConsistentException.java @@ -21,20 +21,20 @@ public class ObTablePartitionConsistentException extends RuntimeException { private int errorCode; - /** + /* * Ob table partition consistent exception. */ public ObTablePartitionConsistentException() { } - /** + /* * Ob table partition consistent exception. */ public ObTablePartitionConsistentException(int errorCode) { this("error code: " + errorCode, errorCode); } - /** + /* * Ob table partition consistent exception. */ public ObTablePartitionConsistentException(String message, int errorCode) { @@ -42,28 +42,28 @@ public ObTablePartitionConsistentException(String message, int errorCode) { this.errorCode = errorCode; } - /** + /* * Ob table partition consistent exception. */ public ObTablePartitionConsistentException(String message) { super(message); } - /** + /* * Ob table partition consistent exception. */ public ObTablePartitionConsistentException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table partition consistent exception. */ public ObTablePartitionConsistentException(Throwable cause) { super(cause); } - /** + /* * Get error code. */ public int getErrorCode() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionInfoRefreshException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionInfoRefreshException.java index cc777a90..416b4876 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionInfoRefreshException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionInfoRefreshException.java @@ -18,41 +18,41 @@ package com.alipay.oceanbase.rpc.exception; public class ObTablePartitionInfoRefreshException extends ObTableEntryRefreshException { - /** + /* * Ob table partition info refresh exception. */ public ObTablePartitionInfoRefreshException() { } - /** + /* * Ob table partition info refresh exception. */ public ObTablePartitionInfoRefreshException(int errorCode) { super(errorCode); } - /** + /* * Ob table partition info refresh exception. */ public ObTablePartitionInfoRefreshException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table partition info refresh exception. */ public ObTablePartitionInfoRefreshException(String message) { super(message); } - /** + /* * Ob table partition info refresh exception. */ public ObTablePartitionInfoRefreshException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table partition info refresh exception. */ public ObTablePartitionInfoRefreshException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionLocationRefreshException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionLocationRefreshException.java index 5b1728ba..4f80df2b 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionLocationRefreshException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionLocationRefreshException.java @@ -18,48 +18,48 @@ package com.alipay.oceanbase.rpc.exception; public class ObTablePartitionLocationRefreshException extends ObTableException { - /** + /* * Ob table partition location refresh exception. */ public ObTablePartitionLocationRefreshException() { } - /** + /* * Ob table partition location refresh exception. */ public ObTablePartitionLocationRefreshException(int errorCode) { super(errorCode); } - /** + /* * Ob table partition location refresh exception. */ public ObTablePartitionLocationRefreshException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table partition location refresh exception. */ public ObTablePartitionLocationRefreshException(String message) { super(message); } - /** + /* * Ob table partition location refresh exception. */ public ObTablePartitionLocationRefreshException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table partition location refresh exception. */ public ObTablePartitionLocationRefreshException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionNoMasterException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionNoMasterException.java index 9c1393d7..a7b98854 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionNoMasterException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionNoMasterException.java @@ -18,48 +18,48 @@ package com.alipay.oceanbase.rpc.exception; public class ObTablePartitionNoMasterException extends ObTablePartitionLocationRefreshException { - /** + /* * Ob table partition no master exception. */ public ObTablePartitionNoMasterException() { } - /** + /* * Ob table partition no master exception. */ public ObTablePartitionNoMasterException(int errorCode) { super(errorCode); } - /** + /* * Ob table partition no master exception. */ public ObTablePartitionNoMasterException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table partition no master exception. */ public ObTablePartitionNoMasterException(String message) { super(message); } - /** + /* * Ob table partition no master exception. */ public ObTablePartitionNoMasterException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table partition no master exception. */ public ObTablePartitionNoMasterException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionNotExistException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionNotExistException.java index bb2a8d90..45ba9184 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionNotExistException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTablePartitionNotExistException.java @@ -18,41 +18,41 @@ package com.alipay.oceanbase.rpc.exception; public class ObTablePartitionNotExistException extends ObTablePartitionLocationRefreshException { - /** + /* * Ob table partition not exist exception. */ public ObTablePartitionNotExistException() { } - /** + /* * Ob table partition not exist exception. */ public ObTablePartitionNotExistException(int errorCode) { super(errorCode); } - /** + /* * Ob table partition not exist exception. */ public ObTablePartitionNotExistException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table partition not exist exception. */ public ObTablePartitionNotExistException(String message) { super(message); } - /** + /* * Ob table partition not exist exception. */ public ObTablePartitionNotExistException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table partition not exist exception. */ public ObTablePartitionNotExistException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableReplicaNotReadableException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableReplicaNotReadableException.java index accb7053..3b1025fa 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableReplicaNotReadableException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableReplicaNotReadableException.java @@ -19,48 +19,48 @@ public class ObTableReplicaNotReadableException extends ObTableException { - /** + /* * Ob replica not readable exception. */ public ObTableReplicaNotReadableException() { } - /** + /* * Ob replica not readable exception. */ public ObTableReplicaNotReadableException(int errorCode) { super(errorCode); } - /** + /* * Ob replica not readable exception. */ public ObTableReplicaNotReadableException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob replica not readable exception. */ public ObTableReplicaNotReadableException(String message) { super(message); } - /** + /* * Ob replica not readable exception. */ public ObTableReplicaNotReadableException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob replica not readable exception. */ public ObTableReplicaNotReadableException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableRetryExhaustedException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableRetryExhaustedException.java index dd495bf2..917033a5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableRetryExhaustedException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableRetryExhaustedException.java @@ -18,41 +18,41 @@ package com.alipay.oceanbase.rpc.exception; public class ObTableRetryExhaustedException extends ObTableException { - /** + /* * Ob table retry exhausted exception. */ public ObTableRetryExhaustedException() { } - /** + /* * Ob table retry exhausted exception. */ public ObTableRetryExhaustedException(int errorCode) { super(errorCode); } - /** + /* * Ob table retry exhausted exception. */ public ObTableRetryExhaustedException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table retry exhausted exception. */ public ObTableRetryExhaustedException(String message) { super(message); } - /** + /* * Ob table retry exhausted exception. */ public ObTableRetryExhaustedException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table retry exhausted exception. */ public ObTableRetryExhaustedException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableRoutingWrongException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableRoutingWrongException.java index 9dadbfa4..5b07eafc 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableRoutingWrongException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableRoutingWrongException.java @@ -18,48 +18,48 @@ package com.alipay.oceanbase.rpc.exception; public class ObTableRoutingWrongException extends ObTableException { - /** + /* * Ob table routing wrong exception. */ public ObTableRoutingWrongException() { } - /** + /* * Ob table routing wrong exception with error code. */ public ObTableRoutingWrongException(int errorCode) { super(errorCode); } - /** + /* * Ob table routing wrong exception with message and error code. */ public ObTableRoutingWrongException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table routing wrong exception with message. */ public ObTableRoutingWrongException(String message) { super(message); } - /** + /* * Ob table routing wrong exception with message and cause. */ public ObTableRoutingWrongException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table routing wrong exception with cause. */ public ObTableRoutingWrongException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerCacheExpiredException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerCacheExpiredException.java index 5b6e929c..b7b8352b 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerCacheExpiredException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerCacheExpiredException.java @@ -18,48 +18,48 @@ package com.alipay.oceanbase.rpc.exception; public class ObTableServerCacheExpiredException extends ObTableException { - /** + /* * Ob table server cache expired exception. */ public ObTableServerCacheExpiredException() { } - /** + /* * Ob table server cache expired exception with error code. */ public ObTableServerCacheExpiredException(int errorCode) { super(errorCode); } - /** + /* * Ob table server cache expired exception with message and error code. */ public ObTableServerCacheExpiredException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table server cache expired exception with message. */ public ObTableServerCacheExpiredException(String message) { super(message); } - /** + /* * Ob table server cache expired exception with message and cause. */ public ObTableServerCacheExpiredException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table server cache expired exception with cause. */ public ObTableServerCacheExpiredException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerConnectException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerConnectException.java index baf74a7f..f37b8db5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerConnectException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerConnectException.java @@ -19,48 +19,48 @@ public class ObTableServerConnectException extends ObTableException { - /** + /* * Ob table server connect exception. */ public ObTableServerConnectException() { } - /** + /* * Ob table server connect exception. */ public ObTableServerConnectException(int errorCode) { super(errorCode); } - /** + /* * Ob table server connect exception. */ public ObTableServerConnectException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table server connect exception. */ public ObTableServerConnectException(String message) { super(message); } - /** + /* * Ob table server connect exception. */ public ObTableServerConnectException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table server connect exception. */ public ObTableServerConnectException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerDownException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerDownException.java index d4b703d6..db72cc55 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerDownException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerDownException.java @@ -19,48 +19,48 @@ public class ObTableServerDownException extends ObTableException { - /** + /* * Ob table server down exception. */ public ObTableServerDownException() { } - /** + /* * Ob table server down exception. */ public ObTableServerDownException(int errorCode) { super(errorCode); } - /** + /* * Ob table server down exception. */ public ObTableServerDownException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table server down exception. */ public ObTableServerDownException(String message) { super(message); } - /** + /* * Ob table server down exception. */ public ObTableServerDownException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table server down exception. */ public ObTableServerDownException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerStatusChangeException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerStatusChangeException.java index a4192827..c7240b9d 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerStatusChangeException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerStatusChangeException.java @@ -19,48 +19,48 @@ public class ObTableServerStatusChangeException extends ObTableException { - /** + /* * Ob table server status change exception. */ public ObTableServerStatusChangeException() { } - /** + /* * Ob table server status change exception. */ public ObTableServerStatusChangeException(int errorCode) { super(errorCode); } - /** + /* * Ob table server status change exception. */ public ObTableServerStatusChangeException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table server status change exception. */ public ObTableServerStatusChangeException(String message) { super(message); } - /** + /* * Ob table server status change exception. */ public ObTableServerStatusChangeException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table server status change exception. */ public ObTableServerStatusChangeException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerTimeoutException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerTimeoutException.java index 34945f3e..7333fbf6 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerTimeoutException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableServerTimeoutException.java @@ -19,41 +19,41 @@ public class ObTableServerTimeoutException extends ObTableException { - /** + /* * Ob table server timeout exception. */ public ObTableServerTimeoutException() { } - /** + /* * Ob table server timeout exception. */ public ObTableServerTimeoutException(int errorCode) { super(errorCode); } - /** + /* * Ob table server timeout exception. */ public ObTableServerTimeoutException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table server timeout exception. */ public ObTableServerTimeoutException(String message) { super(message); } - /** + /* * Ob table server timeout exception. */ public ObTableServerTimeoutException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table server timeout exception. */ public ObTableServerTimeoutException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTenantNotInServerException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTenantNotInServerException.java new file mode 100644 index 00000000..d19e774b --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTenantNotInServerException.java @@ -0,0 +1,69 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.exception; + +public class ObTableTenantNotInServerException extends ObTableException { + + /* + * Ob table tenant not in server exception. + */ + public ObTableTenantNotInServerException() { + } + + /* + * Ob table tenant not in server exception. + */ + public ObTableTenantNotInServerException(int errorCode) { + super(errorCode); + } + + /* + * Ob table tenant not in server exception. + */ + public ObTableTenantNotInServerException(String message, int errorCode) { + super(message, errorCode); + } + + /* + * Ob table tenant not in server exception. + */ + public ObTableTenantNotInServerException(String message) { + super(message); + } + + /* + * Ob table tenant not in server exception. + */ + public ObTableTenantNotInServerException(String message, Throwable cause) { + super(message, cause); + } + + /* + * Ob table tenant not in server exception. + */ + public ObTableTenantNotInServerException(Throwable cause) { + super(cause); + } + + /* + * Is need refresh table entry. + */ + public boolean isNeedRefreshTableEntry() { + return false; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTimeoutExcetion.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTimeoutExcetion.java index 21571402..f9ebb666 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTimeoutExcetion.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTimeoutExcetion.java @@ -18,41 +18,41 @@ package com.alipay.oceanbase.rpc.exception; public class ObTableTimeoutExcetion extends ObTableException { - /** + /* * Ob table timeout excetion. */ public ObTableTimeoutExcetion() { } - /** + /* * Ob table timeout excetion. */ public ObTableTimeoutExcetion(int errorCode) { super(errorCode); } - /** + /* * Ob table timeout excetion. */ public ObTableTimeoutExcetion(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table timeout excetion. */ public ObTableTimeoutExcetion(String message) { super(message); } - /** + /* * Ob table timeout excetion. */ public ObTableTimeoutExcetion(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table timeout excetion. */ public ObTableTimeoutExcetion(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTransactionRpcTimeout.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTransactionRpcTimeout.java index a6ba9797..b105af32 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTransactionRpcTimeout.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTransactionRpcTimeout.java @@ -19,48 +19,48 @@ public class ObTableTransactionRpcTimeout extends ObTableException { - /** + /* * Ob table transaction rpc timeout. */ public ObTableTransactionRpcTimeout() { } - /** + /* * Ob table transaction rpc timeout. */ public ObTableTransactionRpcTimeout(int errorCode) { super(errorCode); } - /** + /* * Ob table transaction rpc timeout. */ public ObTableTransactionRpcTimeout(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table transaction rpc timeout. */ public ObTableTransactionRpcTimeout(String message) { super(message); } - /** + /* * Ob table transaction rpc timeout. */ public ObTableTransactionRpcTimeout(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table transaction rpc timeout. */ public ObTableTransactionRpcTimeout(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTransportException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTransportException.java index ce46235e..3baf5144 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTransportException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableTransportException.java @@ -21,48 +21,48 @@ public class ObTableTransportException extends ObTableException { - /** + /* * Ob table transport exception. */ public ObTableTransportException() { } - /** + /* * Ob table transport exception. */ public ObTableTransportException(int errorCode) { super(errorCode); } - /** + /* * Ob table transport exception. */ public ObTableTransportException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table transport exception. */ public ObTableTransportException(String message) { super(message); } - /** + /* * Ob table transport exception. */ public ObTableTransportException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table transport exception. */ public ObTableTransportException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableUnexpectedException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableUnexpectedException.java index c98d597d..f0088a5a 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableUnexpectedException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableUnexpectedException.java @@ -19,42 +19,42 @@ public class ObTableUnexpectedException extends ObTableException { - /** + /* * Ob table unexpected exception. */ public ObTableUnexpectedException() { } - /** + /* * Ob table unexpected exception. */ public ObTableUnexpectedException(int errorCode) { super(errorCode); } - /** + /* * Ob table unexpected exception. */ public ObTableUnexpectedException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table unexpected exception. */ public ObTableUnexpectedException(String message) { super(message); } - /** + /* * Ob table unexpected exception. */ public ObTableUnexpectedException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table unexpected exception. */ public ObTableUnexpectedException(Throwable cause) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableUnitMigrateException.java b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableUnitMigrateException.java index 8371ec23..08ea1079 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableUnitMigrateException.java +++ b/src/main/java/com/alipay/oceanbase/rpc/exception/ObTableUnitMigrateException.java @@ -19,48 +19,48 @@ public class ObTableUnitMigrateException extends ObTableException { - /** + /* * Ob table unit migrate exception. */ public ObTableUnitMigrateException() { } - /** + /* * Ob table unit migrate exception. */ public ObTableUnitMigrateException(int errorCode) { super(errorCode); } - /** + /* * Ob table unit migrate exception. */ public ObTableUnitMigrateException(String message, int errorCode) { super(message, errorCode); } - /** + /* * Ob table unit migrate exception. */ public ObTableUnitMigrateException(String message) { super(message); } - /** + /* * Ob table unit migrate exception. */ public ObTableUnitMigrateException(String message, Throwable cause) { super(message, cause); } - /** + /* * Ob table unit migrate exception. */ public ObTableUnitMigrateException(Throwable cause) { super(cause); } - /** + /* * Is need refresh table entry. */ public boolean isNeedRefreshTableEntry() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/filter/ObCompareOp.java b/src/main/java/com/alipay/oceanbase/rpc/filter/ObCompareOp.java new file mode 100644 index 00000000..7af06e69 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/filter/ObCompareOp.java @@ -0,0 +1,98 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.filter; + +public enum ObCompareOp { + /* + * LT -> less than + */ + LT, + + /* + * GT -> greater than + */ + GT, + + /* + * LE -> less or equal than + */ + LE, + + /* + * GE -> greater or equal than + */ + GE, + + /* + * NE -> not equal + */ + NE, + + /* + * EQ -> equal + */ + EQ, + + /* + * IS -> IS NULL + */ + IS, + + /* + * IS_NOT -> IS NOT NULL + */ + IS_NOT; + + /* + * operator to string + */ + public String toString() { + String returnString = null; + // switch sign + switch (this) { + case LT: + returnString = "<"; + break; + case GT: + returnString = ">"; + break; + case LE: + returnString = "<="; + break; + case GE: + returnString = ">="; + break; + case NE: + returnString = "!="; + break; + case EQ: + returnString = "="; + break; + case IS: + returnString = "IS"; + break; + case IS_NOT: + returnString = "IS_NOT"; + break; + default: + returnString = ""; + break; + } + return returnString; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilter.java b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilter.java new file mode 100644 index 00000000..c044f972 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilter.java @@ -0,0 +1,31 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.filter; + +public abstract class ObTableFilter { + + public static final String TABLE_COMPARE_FILTER = "TableCompareFilter"; + + /* + * to string + */ + public String toString() { + return null; + }; + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilterFactory.java b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilterFactory.java new file mode 100644 index 00000000..03c14278 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilterFactory.java @@ -0,0 +1,54 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.filter; + +import java.util.List; + +/** + * ObTableFilterFactory is used to create filter without complex procedure + * Suggest all filter should create by this util class + */ +public class ObTableFilterFactory { + public static ObTableFilterList andList(ObTableFilter... filters) { + return new ObTableFilterList(ObTableFilterList.operator.AND, filters); + } + + public static ObTableFilterList orList(ObTableFilter... filters) { + return new ObTableFilterList(ObTableFilterList.operator.OR, filters); + } + + public static ObTableValueFilter compareVal(ObCompareOp op, String columnName, Object value) { + return new ObTableValueFilter(op, columnName, value); + } + + public static ObTableInFilter in(String columnName, Object... values) { + return new ObTableInFilter(columnName, values); + } + + public static ObTableInFilter in(String columnName, List values) { + return new ObTableInFilter(columnName, values); + } + + public static ObTableNotInFilter notIn(String columnName, Object... values) { + return new ObTableNotInFilter(columnName, values); + } + + public static ObTableNotInFilter notIn(String columnName, List values) { + return new ObTableNotInFilter(columnName, values); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilterList.java b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilterList.java new file mode 100644 index 00000000..7bc21d10 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableFilterList.java @@ -0,0 +1,134 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.filter; + +import com.alipay.oceanbase.rpc.exception.ObTableException; + +import java.util.ArrayList; +import java.util.List; + +public class ObTableFilterList extends ObTableFilter { + public enum operator { + AND, OR; + }; + + private operator op; + private List filters; + + /* + * default constructor with op = AND + */ + public ObTableFilterList() { + op = operator.AND; + filters = new ArrayList(); + } + + /* + * construct with ObCompareOp + */ + public ObTableFilterList(operator op) { + if (null == op) { + throw new ObTableException("operator is null"); + } + this.op = op; + this.filters = new ArrayList(); + } + + /** + * construct with ObCompareOp / ObTableFilter + * @param op filter operation + * @param filters table filters + */ + public ObTableFilterList(operator op, ObTableFilter... filters) { + if (null == op) { + throw new ObTableException("operator is null"); + } + this.op = op; + this.filters = new ArrayList(); + + for (int i = 0; i < filters.length; ++i) { + if (null == filters[i]) { + throw new ObTableException(i + "-th filter is null"); + } + this.filters.add(filters[i]); + } + } + + /* + * add filter + */ + public void addFilter(ObTableFilter... filters) { + for (int i = 0; i < filters.length; ++i) { + if (null == filters[i] || this == filters[i]) { + throw new ObTableException(i + "-th input filter is illegal "); + } + this.filters.add(filters[i]); + } + } + + /* + * get the size of filter list + */ + public int size() { + return filters.size(); + } + + /* + * get the pos-th filter of filter list + */ + public ObTableFilter get(int pos) { + if (pos >= size()) { + throw new ObTableException("pos:" + pos + " is out of range: " + size()); + } + return filters.get(pos); + } + + /* + * to string + */ + public String toString() { + StringBuilder filtersString = new StringBuilder(); + String stringOp; + + if (operator.AND == op) { + stringOp = " && "; + } else { + stringOp = " || "; + } + + for (int i = 0; i < filters.size(); ++i) { + String filterString = filters.get(i).toString(); + if (null == filterString || filterString.isEmpty()) { + continue; + } else { + if (0 != i) { + filtersString.append(stringOp); + } + if (filters.get(i) instanceof ObTableValueFilter) { + filtersString.append(filterString); + } else { + filtersString.append("("); + filtersString.append(filterString); + filtersString.append(")"); + } + } + } + // if filtersString is empty then return "" + return filtersString.toString(); + } +} \ No newline at end of file diff --git a/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableInFilter.java b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableInFilter.java new file mode 100644 index 00000000..fd31e621 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableInFilter.java @@ -0,0 +1,91 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.filter; + +import com.alipay.oceanbase.rpc.exception.ObTableException; + +import java.util.List; + +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.*; + +/** + * columnName's value is in values + * Notice that values should not be empty in ObTableInFilter + */ +public class ObTableInFilter extends ObTableFilter { + private String columnName; + private Object[] values; + + /** + * construct with String / Object[] + * @param columnName columnName + * @param values values + */ + public ObTableInFilter(String columnName, Object... values) { + if (null == columnName || columnName.isEmpty()) { + throw new ObTableException("column name is null"); + } + + if (null == values || values.length == 0) { + throw new ObTableException("in filter values should not be empty"); + } + + this.columnName = columnName; + this.values = values; + } + + /** + * construct with String / List of Object + * @param columnName column name + * @param values values + */ + public ObTableInFilter(String columnName, List values) { + if (null == columnName || columnName.isEmpty()) { + throw new ObTableException("column name is null"); + } + + if (null == values || values.isEmpty()) { + throw new ObTableException("in filter values should not be empty"); + } + + this.columnName = columnName; + this.values = values.toArray(); + } + + /** + * get column name + * @return column name + */ + public String getColumnName() { + return columnName; + } + + /** + * TO STRING + * @return Filter represent the in expression + */ + public String toString() { + ObTableFilterList filterList = orList(); + + for (Object value : values) { + filterList.addFilter(compareVal(ObCompareOp.EQ, columnName, value)); + } + + return filterList.toString(); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableNotInFilter.java b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableNotInFilter.java new file mode 100644 index 00000000..0897166e --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableNotInFilter.java @@ -0,0 +1,84 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.filter; + +import com.alipay.oceanbase.rpc.exception.ObTableException; + +import java.util.List; + +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.andList; +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.compareVal; + +public class ObTableNotInFilter extends ObTableFilter { + private String columnName; + private Object[] values; + + /** + * construct with String / Object[] + * @param columnName columnName + * @param values values + */ + public ObTableNotInFilter(String columnName, Object... values) { + if (null == columnName || columnName.isEmpty()) { + throw new ObTableException("column name is null"); + } + + if (null == values || values.length == 0) { + throw new ObTableException("not in filter values should not be empty"); + } + + this.columnName = columnName; + this.values = values; + } + + /** + * construct with String / List of Object + * @param columnName column name + * @param values values + */ + public ObTableNotInFilter(String columnName, List values) { + if (null == columnName || columnName.isEmpty()) { + throw new ObTableException("column name is null"); + } + + if (null == values || values.isEmpty()) { + throw new ObTableException("not in filter values should not be empty"); + } + + this.columnName = columnName; + this.values = values.toArray(); + } + + /** + * get column name + * @return column name + */ + public String getColumnName() { + return columnName; + } + + public String toString() { + ObTableFilterList filterList = andList(); + + for (Object value : values) { + filterList.addFilter(compareVal(ObCompareOp.NE, columnName, value)); + } + + return filterList.toString(); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableValueFilter.java b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableValueFilter.java new file mode 100644 index 00000000..f9c625d3 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/filter/ObTableValueFilter.java @@ -0,0 +1,96 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.filter; + +import com.alipay.oceanbase.rpc.exception.ObTableException; + +public class ObTableValueFilter extends ObTableFilter { + private ObCompareOp op; + private String columnName; + private Object value; + + /* + * construct with ObCompareOp / String / Object + */ + public ObTableValueFilter(ObCompareOp op, String columnName, Object value) + throws ObTableException { + if (op == ObCompareOp.IS || op == ObCompareOp.IS_NOT) { + if (value != null) { + throw new ObTableException(String.format("the value of compare op %s must be null", + op.toString())); + } + } else if (value == null) { + throw new ObTableException(String.format( + "the value of comparer op %s must not be null", op.toString())); + } + + this.op = op; + this.columnName = columnName; + this.value = value; + } + + /* + * set filter + */ + public void set(ObCompareOp op, String columnName, Object value) throws ObTableException { + if (op == ObCompareOp.IS || op == ObCompareOp.IS_NOT) { + if (value != null) { + throw new ObTableException(String.format("the value of %s must be null", + op.toString())); + } + } else if (value == null) { + throw new ObTableException(String.format("the value of %s must not be null", + op.toString())); + } + this.op = op; + this.columnName = columnName; + this.value = value; + } + + /* + * get column name + */ + public String getColumnName() { + return columnName; + } + + /* + * to string + */ + public String toString() { + StringBuilder filterString = new StringBuilder(); + + // handle empty op / columnName + if (null == op || null == columnName || columnName.isEmpty()) { + return null; + } + + filterString.append(TABLE_COMPARE_FILTER); + filterString.append("("); + filterString.append(op.toString()); + filterString.append(", '"); + filterString.append(columnName); + filterString.append(":"); + if (value != null) { + filterString.append(String.valueOf(value)); + } + filterString.append("')"); + + return filterString.toString(); + } +} \ No newline at end of file diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java index 8e928818..20d657e0 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/LocationUtil.java @@ -17,9 +17,12 @@ package com.alipay.oceanbase.rpc.location; +import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONException; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.parser.ParserConfig; +import com.alipay.oceanbase.rpc.ObClusterTableBatchOps; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.constant.Constants; import com.alipay.oceanbase.rpc.exception.*; import com.alipay.oceanbase.rpc.location.model.*; @@ -30,6 +33,7 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; import com.alipay.oceanbase.rpc.protocol.payload.impl.column.ObGeneratedColumn; import com.alipay.oceanbase.rpc.protocol.payload.impl.column.ObSimpleColumn; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObIndexType; import com.alipay.oceanbase.rpc.protocol.payload.impl.parser.ObGeneratedColumnExpressParser; import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; import org.slf4j.Logger; @@ -44,72 +48,143 @@ import static com.alipay.oceanbase.rpc.location.model.partition.ObPartitionKey.MAX_PARTITION_ELEMENT; import static com.alipay.oceanbase.rpc.location.model.partition.ObPartitionKey.MIN_PARTITION_ELEMENT; import static com.alipay.oceanbase.rpc.util.RandomUtil.getRandomNum; -import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; +import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.*; import static java.lang.String.format; +import static com.alipay.oceanbase.rpc.protocol.payload.Constants.INVALID_LS_ID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class LocationUtil { - private static final Logger logger = TableClientLoggerFactory - .getLogger(LocationUtil.class); + private static final Logger logger = TableClientLoggerFactory + .getLogger(LocationUtil.class); static { ParserConfig.getGlobalInstance().setSafeMode(true); } + private static final String OB_VERSION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ OB_VERSION() AS CLUSTER_VERSION;"; + + private static final String PROXY_INDEX_INFO_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ data_table_id, table_id, index_type FROM oceanbase.__all_virtual_table " + + "where table_name = ?"; + + private static final String PROXY_TABLE_ID_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ table_id from oceanbase.__all_virtual_proxy_schema " + + "where tenant_name = ? and database_name = ? and table_name = ? limit 1"; + + private static final String OB_TENANT_EXIST_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ tenant_id from __all_tenant where tenant_name = ?;"; + + @Deprecated + @SuppressWarnings("unused") + private static final String PROXY_PLAIN_SCHEMA_SQL_FORMAT = "SELECT /*+READ_CONSISTENCY(WEAK)*/ partition_id, svr_ip, sql_port, table_id, role, part_num, replica_num, schema_version, spare1 " + + "FROM oceanbase.__all_virtual_proxy_schema " + + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND partition_id in ({0}) AND sql_port > 0 " + + "ORDER BY role ASC LIMIT ?"; + + private static final String PROXY_PART_INFO_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_level, part_num, part_type, part_space, part_expr, " + + "part_range_type, part_interval_bin, interval_start_bin, " + + "sub_part_num, sub_part_type, sub_part_space, " + + "sub_part_range_type, def_sub_part_interval_bin, def_sub_interval_start_bin, sub_part_expr, " + + "part_key_name, part_key_type, part_key_idx, part_key_extra, spare1 " + + "FROM oceanbase.__all_virtual_proxy_partition_info " + + "WHERE table_id = ? group by part_key_name order by part_key_name LIMIT ?;"; @Deprecated @SuppressWarnings("unused") - private static final String PROXY_PLAIN_SCHEMA_SQL_FORMAT = "SELECT /*+READ_CONSISTENCY(WEAK)*/ partition_id, svr_ip, sql_port, table_id, role, part_num, replica_num, schema_version, spare1 " - + "FROM oceanbase.__all_virtual_proxy_schema " - + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND partition_id in ({0}) AND sql_port > 0 " - + "ORDER BY role ASC LIMIT ?"; - - private static final String PROXY_PART_INFO_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_level, part_num, part_type, part_space, part_expr, " - + "part_range_type, part_interval_bin, interval_start_bin, " - + "sub_part_num, sub_part_type, sub_part_space, " - + "sub_part_range_type, def_sub_part_interval_bin, def_sub_interval_start_bin, sub_part_expr, " - + "part_key_name, part_key_type, part_key_idx, part_key_extra, spare1 " - + "FROM oceanbase.__all_virtual_proxy_partition_info " - + "WHERE table_id = ? group by part_key_name order by part_key_name LIMIT ?;"; + private static final String PROXY_TENANT_SCHEMA_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ svr_ip, sql_port, table_id, role, part_num, replica_num, spare1 " + + "FROM oceanbase.__all_virtual_proxy_schema " + + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND sql_port > 0 " + + "ORDER BY partition_id ASC, role ASC LIMIT ?"; + + private static final String PROXY_DUMMY_LOCATION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ?"; + + private static final String PROXY_LOCATION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ? and partition_id = 0"; + + private static final String PROXY_LOCATION_SQL_PARTITION = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ? and partition_id in ({0})"; + + private static final String PROXY_FIRST_PARTITION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_id, part_name, high_bound_val " + + "FROM oceanbase.__all_virtual_proxy_partition " + + "WHERE table_id = ? LIMIT ?;"; + + private static final String PROXY_SUB_PARTITION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ sub_part_id, part_name, high_bound_val " + + "FROM oceanbase.__all_virtual_proxy_sub_partition " + + "WHERE table_id = ? LIMIT ?;"; + + private static final String PROXY_SERVER_STATUS_INFO = "SELECT ss.svr_ip, ss.zone, zs.region, zs.spare4 as idc " + + "FROM oceanbase.__all_virtual_proxy_server_stat ss, oceanbase.__all_virtual_zone_stat zs " + + "WHERE zs.zone = ss.zone ;"; + @Deprecated @SuppressWarnings("unused") - private static final String PROXY_TENANT_SCHEMA_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ svr_ip, sql_port, table_id, role, part_num, replica_num, spare1 " - + "FROM oceanbase.__all_virtual_proxy_schema " - + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND sql_port > 0 " - + "ORDER BY partition_id ASC, role ASC LIMIT ?"; - - private static final String PROXY_DUMMY_LOCATION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " - + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type " - + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "WHERE tenant_name = ? and database_name=? and table_name = ?"; - - private static final String PROXY_LOCATION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " - + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type " - + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "WHERE tenant_name = ? and database_name=? and table_name = ? and partition_id = 0"; - - private static final String PROXY_LOCATION_SQL_PARTITION = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.partition_id as partition_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " - + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " - + ", A.spare1 as replica_type " - + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " - + "WHERE tenant_name = ? and database_name=? and table_name = ? and partition_id in ({0})"; - - private static final String PROXY_FIRST_PARTITION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_id, part_name, high_bound_val " - + "FROM oceanbase.__all_virtual_proxy_partition " - + "WHERE table_id = ? LIMIT ?;"; - - private static final String PROXY_SUB_PARTITION_SQL = "SELECT /*+READ_CONSISTENCY(WEAK)*/ sub_part_id, part_name, high_bound_val " - + "FROM oceanbase.__all_virtual_proxy_sub_partition " - + "WHERE table_id = ? and part_id = ? LIMIT ?;"; - - private static final String PROXY_SERVER_STATUS_INFO = "SELECT ss.svr_ip, ss.zone, zs.region, zs.spare4 as idc " - + "FROM oceanbase.__all_virtual_proxy_server_stat ss, oceanbase.__all_virtual_zone_stat zs " - + "WHERE zs.zone = ss.zone ;"; - - private static final String home = System.getProperty("user.home", - "/home/admin"); - - private static final int TEMPLATE_PART_ID = -1; + private static final String PROXY_PLAIN_SCHEMA_SQL_FORMAT_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, schema_version, spare1 " + + "FROM oceanbase.__all_virtual_proxy_schema " + + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND tablet_id in ({0}) AND sql_port > 0 " + + "ORDER BY role ASC LIMIT ?"; + + private static final String PROXY_PART_INFO_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_level, part_num, part_type, part_space, part_expr, " + + "part_range_type, sub_part_num, sub_part_type, sub_part_space, sub_part_range_type, sub_part_expr, " + + "part_key_name, part_key_type, part_key_idx, part_key_extra, part_key_collation_type " + + "FROM oceanbase.__all_virtual_proxy_partition_info " + + "WHERE tenant_name = ? and table_id = ? group by part_key_name order by part_key_name LIMIT ?;"; + @Deprecated + @SuppressWarnings("unused") + private static final String PROXY_TENANT_SCHEMA_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ svr_ip, sql_port, table_id, role, part_num, replica_num, spare1 " + + "FROM oceanbase.__all_virtual_proxy_schema " + + "WHERE tenant_name = ? AND database_name = ? AND table_name = ? AND sql_port > 0 " + + "ORDER BY tablet_id ASC, role ASC LIMIT ?"; + + private static final String PROXY_DUMMY_LOCATION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.tablet_id as tablet_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ?"; + + private static final String PROXY_LOCATION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.tablet_id as tablet_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "WHERE tenant_name = ? and database_name=? and table_name = ? and tablet_id = 0"; + + private static final String PROXY_LOCATION_SQL_PARTITION_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ A.tablet_id as tablet_id, A.svr_ip as svr_ip, A.sql_port as sql_port, " + + "A.table_id as table_id, A.role as role, A.replica_num as replica_num, A.part_num as part_num, B.svr_port as svr_port, B.status as status, B.stop_time as stop_time " + + ", A.spare1 as replica_type, D.ls_id as ls_id " + + "FROM oceanbase.__all_virtual_proxy_schema A inner join oceanbase.__all_server B on A.svr_ip = B.svr_ip and A.sql_port = B.inner_port " + + "inner join oceanbase.DBA_OB_TENANTS C on C.tenant_name = A.tenant_name " + + "left join oceanbase.CDB_OB_TABLET_TO_LS D on D.tenant_id = C.tenant_id and D.tablet_id = A.tablet_id " + + "WHERE C.tenant_name = ? and database_name= ? and table_name = ? and A.tablet_id in ({0}) "; + + private static final String PROXY_FIRST_PARTITION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ part_id, part_name, tablet_id, high_bound_val, sub_part_num " + + "FROM oceanbase.__all_virtual_proxy_partition " + + "WHERE tenant_name = ? and table_id = ? LIMIT ?;"; + + private static final String PROXY_SUB_PARTITION_SQL_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ sub_part_id, part_name, tablet_id, high_bound_val " + + "FROM oceanbase.__all_virtual_proxy_sub_partition " + + "WHERE tenant_name = ? and table_id = ? LIMIT ?;"; + + private static final String PROXY_SERVER_STATUS_INFO_V4 = "SELECT ss.svr_ip, ss.zone, zs.region, zs.idc as idc " + + "FROM DBA_OB_SERVERS ss, DBA_OB_ZONES zs " + + "WHERE zs.zone = ss.zone ;"; + + private static final String home = System.getProperty("user.home", + "/home/admin"); + + private static final String TABLE_GROUP_GET_TABLE_NAME_V4 = "SELECT /*+READ_CONSISTENCY(WEAK)*/ table_name " + + "FROM oceanbase.CDB_OB_TABLEGROUP_TABLES " + + "WHERE tablegroup_name = ? and tenant_id = ? limit 1;"; + + private static final int TEMPLATE_PART_ID = -1; + + // limit the size of get tableEntry location from remote each time + private static final int MAX_TABLET_NUMS_EPOCH = 300; private abstract static class TableEntryRefreshWithPriorityCallback { abstract T execute(ObServerAddr obServerAddr) throws ObTableEntryRefreshException; @@ -134,12 +209,16 @@ private static TableEntry callTableEntryRefreshWithPriority(ServerRoster serverR serverRoster.resetPriority(addr); return tableEntry; } catch (ObTableEntryRefreshException e) { + RUNTIME.error("callTableEntryRefreshWithPriority meet exception", e); serverRoster.downgradePriority(addr); throw e; + } catch (Throwable t) { + RUNTIME.error("callTableEntryRefreshWithPriority meet exception", t); + throw t; } } - /** + /* * Get Server LDC info from OB system table. */ public static List getServerLdc(ServerRoster serverRoster, @@ -156,12 +235,16 @@ public static List getServerLdc(ServerRoster serverRoster, serverRoster.resetPriority(addr); return ss; } catch (ObTableEntryRefreshException e) { + RUNTIME.error("getServerLdc meet exception", e); serverRoster.downgradePriority(addr); throw e; + } catch (Throwable t) { + RUNTIME.error("callTableEntryRefreshWithPriority meet exception", t); + throw t; } } - /** + /* * Format ob server url with the tenant server. * * @param obServerAddr @@ -176,7 +259,7 @@ private static String formatObServerUrl(ObServerAddr obServerAddr, long connectT obServerAddr.getIp() + ":" + obServerAddr.getSqlPort(), connectTimeout, socketTimeout); } - /** + /* * Establish db connection to the given database URL, with proxy user/password. * * @param url @@ -185,21 +268,35 @@ private static String formatObServerUrl(ObServerAddr obServerAddr, long connectT */ private static Connection getMetaRefreshConnection(String url, ObUserAuth sysUA) throws ObTableEntryRefreshException { + loadJdbcDriver(); + try { - Class.forName("com.mysql.jdbc.Driver"); return DriverManager.getConnection(url, sysUA.getUserName(), sysUA.getPassword()); - } catch (ClassNotFoundException e) { - logger.error(LCD.convert("01-00006"), e.getMessage(), e); - throw new ObTableEntryRefreshException(format( - "fail to find com.mysql.jdbc.Driver, errMsg=%s", e.getMessage()), e); } catch (Exception e) { - logger.error(LCD.convert("01-00005"), e.getMessage(), e); - throw new ObTableEntryRefreshException("fail to decode proxyro password", e); + RUNTIME.error(LCD.convert("01-00005"), e.getMessage(), e); + throw new ObTableEntryRefreshException("fail to connect meta server", e); } + } + private static void loadJdbcDriver() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + return; + } catch (ClassNotFoundException e) { + RUNTIME.info("Class 'com.mysql.cj.jdbc.Driver' not found, " + + "try to load legacy driver class 'com.mysql.jdbc.Driver'"); + } + + try { + Class.forName("com.mysql.jdbc.Driver"); + } catch (ClassNotFoundException e) { + RUNTIME.error(LCD.convert("01-00006"), e.getMessage(), e); + throw new ObTableEntryRefreshException(format("fail to find jdbc driver, errMsg=%s", + e.getMessage()), e); + } } - /** + /* * Refresh server LDC info. * * @param obServerAddr @@ -219,7 +316,11 @@ private static List callServerLdcRefresh(ObServerAddr obServerA ResultSet rs = null; try { connection = getMetaRefreshConnection(url, sysUA); - ps = connection.prepareStatement(PROXY_SERVER_STATUS_INFO); + if (ObGlobal.obVsnMajor() >= 4) { + ps = connection.prepareStatement(PROXY_SERVER_STATUS_INFO_V4); + } else { + ps = connection.prepareStatement(PROXY_SERVER_STATUS_INFO); + } rs = ps.executeQuery(); while (rs.next()) { String ip = rs.getString("svr_ip"); @@ -229,7 +330,7 @@ private static List callServerLdcRefresh(ObServerAddr obServerA ss.add(new ObServerLdcItem(ip, zone, idc, region)); } } catch (Exception e) { - logger.error(LCD.convert("01-00027"), url, e); + RUNTIME.error(LCD.convert("01-00027"), url, e); throw new ObTableEntryRefreshException(format( "fail to refresh server LDC from remote url=%s", url), e); } finally { @@ -253,7 +354,7 @@ private static List callServerLdcRefresh(ObServerAddr obServerA private static TableEntry callTableEntryRefresh(ObServerAddr obServerAddr, TableEntryKey key, long connectTimeout, long socketTimeout, - ObUserAuth sysUA, + ObUserAuth sysUA, boolean initialized, TableEntryRefreshCallback callback) throws ObTableEntryRefreshException { String url = formatObServerUrl(obServerAddr, connectTimeout, socketTimeout); @@ -264,9 +365,14 @@ private static TableEntry callTableEntryRefresh(ObServerAddr obServerAddr, Table entry = callback.execute(connection); } catch (ObTableNotExistException e) { // avoid to refresh meta for ObTableNotExistException + RUNTIME.error("callTableEntryRefresh meet exception", e); throw e; } catch (Exception e) { - logger.error(LCD.convert("01-00007"), url, key, e); + if (!initialized) { + BOOT.error(LCD.convert("01-00007"), url, key, e); + } else { + RUNTIME.error(LCD.convert("01-00007"), url, key, e); + } throw new ObTableEntryRefreshException(format( "fail to refresh table entry from remote url=%s, key=%s", url, key), e); } finally { @@ -283,14 +389,19 @@ private static TableEntry callTableEntryRefresh(ObServerAddr obServerAddr, Table entry.setRefreshTimeMills(System.currentTimeMillis()); return entry; } else { - logger.error(LCD.convert("01-00008"), obServerAddr, key, entry); + if (!initialized) { + BOOT.error(LCD.convert("01-00008"), obServerAddr, key, entry); + } else { + RUNTIME.error(LCD.convert("01-00008"), obServerAddr, key, entry); + RUNTIME.error("table entry is invalid"); + } throw new ObTableEntryRefreshException("table entry is invalid, addr = " + obServerAddr + " key =" + key + " entry =" + entry); } } - /** + /* * Load table entry with priority. */ public static TableEntry loadTableEntryWithPriority(final ServerRoster serverRoster, @@ -306,18 +417,18 @@ public static TableEntry loadTableEntryWithPriority(final ServerRoster serverRos @Override TableEntry execute(ObServerAddr obServerAddr) throws ObTableEntryRefreshException { return callTableEntryRefresh(obServerAddr, key, connectTimeout, socketTimeout, - sysUA, new TableEntryRefreshCallback() { + sysUA, true, new TableEntryRefreshCallback() { @Override TableEntry execute(Connection connection) throws ObTableEntryRefreshException { - return getTableEntryFromRemote(connection, key); + return getTableEntryFromRemote(connection, key, true); } }); } }); } - /** + /* * Load table entry location with priority. */ public static TableEntry loadTableEntryLocationWithPriority(final ServerRoster serverRoster, @@ -335,7 +446,7 @@ public static TableEntry loadTableEntryLocationWithPriority(final ServerRoster s @Override TableEntry execute(ObServerAddr obServerAddr) throws ObTableEntryRefreshException { return callTableEntryRefresh(obServerAddr, key, connectTimeout, socketTimeout, - sysUA, new TableEntryRefreshCallback() { + sysUA, true, new TableEntryRefreshCallback() { @Override TableEntry execute(Connection connection) throws ObTablePartitionLocationRefreshException { @@ -346,46 +457,213 @@ TableEntry execute(Connection connection) }); } - /** + /* + * load Table Name With table Group + */ + public static String loadTableNameWithGroupName(final ServerRoster serverRoster, + final TableEntryKey key, + final long connectTimeout, + final long socketTimeout, + final long priorityTimeout, + final long cachingTimeout, + final ObUserAuth sysUA) + throws ObTableNotExistException { + Connection connection = null; + String realTableName = ""; + String url = ""; + ObServerAddr addr = serverRoster.getServer(priorityTimeout, cachingTimeout); + try { + url = formatObServerUrl(addr, connectTimeout, socketTimeout); + connection = getMetaRefreshConnection(url, sysUA); + realTableName = getTableNameByGroupNameFromRemote(connection, key); + serverRoster.resetPriority(addr); + } catch (ObTableNotExistException e) { + RUNTIME.error("callTableEntryNameWithPriority meet exception", e); + serverRoster.downgradePriority(addr); + throw e; + } catch (Exception e) { + throw new ObTableNotExistException(format( + "fail to get table name from remote url=%s, key=%s", url, key), e); + } catch (Throwable t) { + RUNTIME.error("callTableEntryNameWithPriority meet exception", t); + throw t; + } finally { + try { + if (null != connection) { + connection.close(); + } + } catch (SQLException e) { + // ignore + } + } + if (realTableName != null && !realTableName.isEmpty()) { + return realTableName; + } else { + throw new ObTableNotExistException("table name is invalid, addr = " + addr + " key =" + + key + " tableName =" + realTableName); + } + + } + + /* + * get TableName From Remote with Group + */ + private static String getTableNameByGroupNameFromRemote(Connection connection, TableEntryKey key) + throws ObTableNotExistException { + PreparedStatement ps = null; + ResultSet rs = null; + String realTableName = ""; + int tenantId = -1; + try { + if (ObGlobal.obVsnMajor() == 0) { + getObVersionFromRemote(connection); + } + tenantId = checkTenantExistFromRemote(connection, key); + if (ObGlobal.obVsnMajor() >= 4) { + ps = connection.prepareStatement(TABLE_GROUP_GET_TABLE_NAME_V4); + ps.setString(1, key.getTableName()); + ps.setString(2, String.valueOf(tenantId)); + } else { + throw new ObTableNotExistException(format( + "fail to get table name from remote in low version than 4, key=%s", key)); + } + rs = ps.executeQuery(); + while (rs.next()) { + realTableName = rs.getString("table_name"); + } + } catch (ObTableNotExistException e) { + // avoid to refresh meta for ObTableNotExistException + RUNTIME.error("getTableNameByGroupNameFromRemote meet exception", e); + throw e; + } catch (Exception e) { + RUNTIME.error("getTableNameByGroupNameFromRemote meet exception", e); + throw new ObTableNotExistException(format("fail to get table name from remote, key=%s", + key), e); + } finally { + try { + if (null != rs) { + rs.close(); + } + if (null != ps) { + ps.close(); + } + } catch (SQLException e) { + // ignore + } + } + return realTableName; + } + + /* * Load table entry randomly. */ public static TableEntry loadTableEntryRandomly(final List rsList,// final TableEntryKey key, // final long connectTimeout,// - final long socketTimeout, final ObUserAuth sysUA) - throws ObTableEntryRefreshException { + final long socketTimeout, + final ObUserAuth sysUA, + final boolean initialized) + throws ObTableEntryRefreshException { return callTableEntryRefresh(randomObServers(rsList), key, connectTimeout, socketTimeout, - sysUA, new TableEntryRefreshCallback() { + sysUA, initialized, new TableEntryRefreshCallback() { @Override TableEntry execute(Connection connection) throws ObTableEntryRefreshException { - return getTableEntryFromRemote(connection, key); + return getTableEntryFromRemote(connection, key, initialized); } }); } - private static TableEntry getTableEntryFromRemote(Connection connection, TableEntryKey key) - throws ObTableEntryRefreshException { + private static void getObVersionFromRemote(Connection connection) + throws ObTableEntryRefreshException { + PreparedStatement ps = null; + ResultSet rs = null; + try { + ps = connection.prepareStatement(OB_VERSION_SQL); + rs = ps.executeQuery(); + if (rs.next()) { + String versionString = rs.getString("CLUSTER_VERSION"); + parseObVersionFromSQL(versionString); + } else { + throw new ObTableEntryRefreshException("fail to get ob version from remote"); + } + } catch (Exception e) { + throw new ObTableEntryRefreshException("fail to get ob version from remote", e); + } finally { + try { + if (null != rs) { + rs.close(); + } + if (null != ps) { + ps.close(); + } + } catch (SQLException e) { + // ignore + } + } + } + + // check tenant exist or not + private static int checkTenantExistFromRemote(Connection connection, TableEntryKey key) + throws ObTableEntryRefreshException { + try (PreparedStatement ps = connection.prepareStatement(OB_TENANT_EXIST_SQL)) { + ps.setString(1, key.getTenantName()); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + throw new ObTableEntryRefreshException("fail to get tenant id from remote"); + } else { + return rs.getInt("tenant_id"); + } + } catch (Exception e) { + throw new ObTableEntryRefreshException("fail to get tenant id from remote", e); + } + } catch (Exception e) { + throw new ObTableEntryRefreshException("fail to get tenant id from remote", e); + } + } + + private static TableEntry getTableEntryFromRemote(Connection connection, TableEntryKey key, + boolean initialized) + throws ObTableEntryRefreshException { PreparedStatement ps = null; ResultSet rs = null; TableEntry tableEntry; + int tenantId = -1; try { - if (key.getTableName().equals(Constants.ALL_DUMMY_TABLE)) { - ps = connection.prepareStatement(PROXY_DUMMY_LOCATION_SQL); - ps.setString(1, key.getTenantName()); - ps.setString(2, key.getDatabaseName()); - ps.setString(3, key.getTableName()); + if (ObGlobal.obVsnMajor() == 0) { + getObVersionFromRemote(connection); + } + tenantId = checkTenantExistFromRemote(connection, key); + if (ObGlobal.obVsnMajor() >= 4) { + if (key.getTableName().equals(Constants.ALL_DUMMY_TABLE)) { + ps = connection.prepareStatement(PROXY_DUMMY_LOCATION_SQL_V4); + ps.setString(1, key.getTenantName()); + ps.setString(2, key.getDatabaseName()); + ps.setString(3, key.getTableName()); + } else { + ps = connection.prepareStatement(PROXY_LOCATION_SQL_V4); + ps.setString(1, key.getTenantName()); + ps.setString(2, key.getDatabaseName()); + ps.setString(3, key.getTableName()); + } } else { - ps = connection.prepareStatement(PROXY_LOCATION_SQL); - ps.setString(1, key.getTenantName()); - ps.setString(2, key.getDatabaseName()); - ps.setString(3, key.getTableName()); + if (key.getTableName().equals(Constants.ALL_DUMMY_TABLE)) { + ps = connection.prepareStatement(PROXY_DUMMY_LOCATION_SQL); + ps.setString(1, key.getTenantName()); + ps.setString(2, key.getDatabaseName()); + ps.setString(3, key.getTableName()); + } else { + ps = connection.prepareStatement(PROXY_LOCATION_SQL); + ps.setString(1, key.getTenantName()); + ps.setString(2, key.getDatabaseName()); + ps.setString(3, key.getTableName()); + } } rs = ps.executeQuery(); tableEntry = getTableEntryFromResultSet(key, rs); if (null != tableEntry) { tableEntry.setTableEntryKey(key); - getTableEntryLocationFromRemote(connection, key, tableEntry); // TODO: check capacity flag later + // fetch tablet ids when table is partition table if (tableEntry.isPartitionTable()) { // fetch partition info fetchPartitionInfo(connection, tableEntry); @@ -394,33 +672,36 @@ private static TableEntry getTableEntryFromRemote(Connection connection, TableEn if (null != tableEntry.getPartitionInfo().getFirstPartDesc()) { ObPartFuncType obPartFuncType = tableEntry.getPartitionInfo() .getFirstPartDesc().getPartFuncType(); - if (obPartFuncType.isRangePart() || obPartFuncType.isListPart()) { - fetchFirstPart(connection, tableEntry, obPartFuncType); - } + fetchFirstPart(connection, tableEntry, obPartFuncType); } // fetch sub range part if (null != tableEntry.getPartitionInfo().getSubPartDesc()) { ObPartFuncType subPartFuncType = tableEntry.getPartitionInfo() .getSubPartDesc().getPartFuncType(); - if (subPartFuncType.isRangePart() || subPartFuncType.isListPart()) { - fetchSubPart(connection, tableEntry, subPartFuncType); - } + fetchSubPart(connection, tableEntry, subPartFuncType); } - // build partition name-id map - tableEntry.getPartitionInfo().setPartNameIdMap( - buildPartNameIdMap(tableEntry.getPartitionInfo())); } } - if (logger.isInfoEnabled()) { - logger.info("get table entry from remote, entry={}", tableEntry); + // get location info + getTableEntryLocationFromRemote(connection, key, tableEntry); + + if (!initialized) { + if (BOOT.isInfoEnabled()) { + BOOT.info("get table entry from remote, entry={}", JSON.toJSON(tableEntry)); + } + } else { + if (logger.isInfoEnabled()) { + logger.info("get table entry from remote"); + } } } } catch (ObTableNotExistException e) { // avoid to refresh meta for ObTableNotExistException + RUNTIME.error("getTableEntryFromRemote meet exception", e); throw e; } catch (Exception e) { - logger.error(LCD.convert("01-00009"), key, e); + RUNTIME.error(LCD.convert("01-00009"), key, e); throw new ObTableEntryRefreshException(format( "fail to get table entry from remote, key=%s", key), e); } finally { @@ -438,7 +719,64 @@ private static TableEntry getTableEntryFromRemote(Connection connection, TableEn return tableEntry; } - /** + private static String genLocationSQLByOffset(TableEntry tableEntry, int offset, int size) { + StringBuilder sb = new StringBuilder(); + String sql = null; + long endOffset = -1; + long allPartNum = tableEntry.getPartitionNum(); + if (offset < 0 || offset >= allPartNum || size < 0) { + throw new IllegalArgumentException("Illegal arguement: offset: " + offset + ", size: " + + size); + } else { + endOffset = Math.min(offset + size, allPartNum); + } + + if (ObGlobal.obVsnMajor() >= 4) { + if (tableEntry.isPartitionTable()) { + Map partTabletIdMap = tableEntry.getPartitionInfo() + .getPartTabletIdMap(); + Long[] tabletIds = partTabletIdMap.values().toArray(new Long[0]); + for (int i = offset; i < endOffset; i++) { + if (i > offset) { + sb.append(", "); + } + sb.append(tabletIds[i]); + } + } else { + for (int i = offset; i < endOffset; i++) { + if (i > offset) { + sb.append(", "); + } + sb.append(i); + } + } + sql = MessageFormat.format(PROXY_LOCATION_SQL_PARTITION_V4, sb.toString()); + } else { + if (tableEntry.isPartitionTable() + && null != tableEntry.getPartitionInfo().getSubPartDesc()) { + long subPartNum = tableEntry.getPartitionInfo().getSubPartDesc().getPartNum(); + for (long i = offset; i < endOffset; ++i) { + if (i > offset) { + sb.append(", "); + } + Long firstPartId = i / subPartNum; + Long subPartId = i % subPartNum; + sb.append(ObPartIdCalculator.generatePartId(firstPartId, subPartId)); + } + } else { + for (int i = offset; i < endOffset; i++) { + if (i > offset) { + sb.append(", "); + } + sb.append(i); + } + } + sql = MessageFormat.format(PROXY_LOCATION_SQL_PARTITION, sb.toString()); + } + return sql; + } + + /* * Get table entry location from remote. */ public static TableEntry getTableEntryLocationFromRemote(Connection connection, @@ -448,32 +786,108 @@ public static TableEntry getTableEntryLocationFromRemote(Connection connection, PreparedStatement ps = null; ResultSet rs = null; + ObPartitionEntry partitionEntry = new ObPartitionEntry(); long partitionNum = tableEntry.getPartitionNum(); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < partitionNum; i++) { - if (i > 0) { - sb.append(", "); + int epoch = (int) ((partitionNum / MAX_TABLET_NUMS_EPOCH) + 1); + for (int i = 0; i < epoch; i++) { + try { + int offset = i * MAX_TABLET_NUMS_EPOCH; + String sql = genLocationSQLByOffset(tableEntry, offset, MAX_TABLET_NUMS_EPOCH); + ps = connection.prepareStatement(sql); + ps.setString(1, key.getTenantName()); + ps.setString(2, key.getDatabaseName()); + ps.setString(3, key.getTableName()); + rs = ps.executeQuery(); + partitionEntry = getPartitionLocationFromResultSet(tableEntry, rs, partitionEntry); + } catch (Exception e) { + RUNTIME.error(LCD.convert("01-00010"), key, partitionNum, tableEntry, e); + throw new ObTablePartitionLocationRefreshException(format( + "fail to get partition location entry from remote entryKey = %s partNum = %d tableEntry =%s " + + "offset =%d epoch =%d", key, partitionNum, tableEntry, i, epoch), e); + } finally { + try { + if (null != rs) { + rs.close(); + } + if (null != ps) { + ps.close(); + } + } catch (SQLException e) { + // ignore + } } - sb.append(i); - } - ObPartitionEntry partitionEntry; - String sql = MessageFormat.format(PROXY_LOCATION_SQL_PARTITION, sb.toString()); + } // end for + tableEntry.setPartitionEntry(partitionEntry); + tableEntry.setRefreshTimeMills(System.currentTimeMillis()); + return tableEntry; + } + + public static Long getTableIdFromRemote(ObServerAddr obServerAddr, ObUserAuth sysUA, + long connectTimeout, long socketTimeout, + String tenantName, String databaseName, String tableName) + throws ObTableEntryRefreshException { + Long tableId = null; + Connection connection = null; + PreparedStatement ps = null; + ResultSet rs = null; try { - ps = connection.prepareStatement(sql); - ps.setString(1, key.getTenantName()); - ps.setString(2, key.getDatabaseName()); - ps.setString(3, key.getTableName()); + String url = formatObServerUrl(obServerAddr, connectTimeout, socketTimeout); + connection = getMetaRefreshConnection(url, sysUA); + ps = connection.prepareStatement(PROXY_TABLE_ID_SQL); + ps.setString(1, tenantName); + ps.setString(2, databaseName); + ps.setString(3, tableName); + rs = ps.executeQuery(); + if (rs.next()) { + tableId = rs.getLong("table_id"); + } else { + throw new ObTableEntryRefreshException("fail to get " + tableName + + " table_id from remote"); + } + } catch (Exception e) { + throw new ObTableEntryRefreshException("fail to get " + tableName + + " table_id from remote", e); + } finally { + try { + if (null != rs) { + rs.close(); + } + if (null != ps) { + ps.close(); + } + } catch (SQLException e) { + // ignore + } + } + return tableId; + } + public static ObIndexInfo getIndexInfoFromRemote(ObServerAddr obServerAddr, ObUserAuth sysUA, + long connectTimeout, long socketTimeout, + String indexTableName) + throws ObTableEntryRefreshException { + ObIndexInfo indexInfo = null; + Connection connection = null; + PreparedStatement ps = null; + ResultSet rs = null; + try { + String url = formatObServerUrl(obServerAddr, connectTimeout, socketTimeout); + connection = getMetaRefreshConnection(url, sysUA); + ps = connection.prepareStatement(PROXY_INDEX_INFO_SQL); + ps.setString(1, indexTableName); rs = ps.executeQuery(); - partitionEntry = getPartitionLocationFromResultSet(tableEntry, rs); - tableEntry.setPartitionEntry(partitionEntry); - tableEntry.setRefreshTimeMills(System.currentTimeMillis()); + if (rs.next()) { + indexInfo = new ObIndexInfo(); + indexInfo.setDataTableId(rs.getLong("data_table_id")); + indexInfo.setIndexTableId(rs.getLong("table_id")); + indexInfo.setIndexType(ObIndexType.valueOf(rs.getInt("index_type"))); + } else { + throw new ObTableEntryRefreshException( + "fail to get index info from remote, result set is empty"); + } } catch (Exception e) { - logger.error(LCD.convert("01-00010"), key, partitionNum, tableEntry, e); - throw new ObTablePartitionLocationRefreshException( - format( - "fail to get partition location entry from remote entryKey = %s partNum = %d tableEntry =%s", - key, partitionNum, tableEntry), e); + throw new ObTableEntryRefreshException(format( + "fail to get index info from remote, indexTableName: %s", indexTableName), e); } finally { try { if (null != rs) { @@ -486,47 +900,65 @@ public static TableEntry getTableEntryLocationFromRemote(Connection connection, // ignore } } - return tableEntry; + return indexInfo; } private static void fetchFirstPart(Connection connection, TableEntry tableEntry, ObPartFuncType obPartFuncType) throws ObTablePartitionInfoRefreshException { + String tableName = ""; + TableEntryKey key = tableEntry.getTableEntryKey(); + if (key != null) { + tableName = key.getDatabaseName() + "." + key.getTableName(); + } + String uuid = UUID.randomUUID().toString(); + PreparedStatement ps = null; ResultSet rs = null; try { - ps = connection.prepareStatement(PROXY_FIRST_PARTITION_SQL); - ps.setLong(1, tableEntry.getTableId()); - ps.setInt(2, Integer.MAX_VALUE); - + if (ObGlobal.obVsnMajor() >= 4) { + ps = connection.prepareStatement(PROXY_FIRST_PARTITION_SQL_V4); + ps.setString(1, key.getTenantName()); + ps.setLong(2, tableEntry.getTableId()); + ps.setInt(3, Integer.MAX_VALUE); + } else { + ps = connection.prepareStatement(PROXY_FIRST_PARTITION_SQL); + ps.setLong(1, tableEntry.getTableId()); + ps.setInt(2, Integer.MAX_VALUE); + } rs = ps.executeQuery(); if (obPartFuncType.isRangePart()) { + List> highBoundVals = new ArrayList<>(); List> bounds = parseFirstPartRange(rs, - tableEntry); + tableEntry, highBoundVals); ((ObRangePartDesc) tableEntry.getPartitionInfo().getFirstPartDesc()) .setBounds(bounds); + ((ObRangePartDesc) tableEntry.getPartitionInfo().getFirstPartDesc()) + .setHighBoundValues(highBoundVals); if (logger.isInfoEnabled()) { - logger.info(format("get first ranges from remote, bounds=%s for tableEntry=%s", - bounds, tableEntry)); + logger.info(format("uuid:%s, get first ranges from remote for %s, bounds=%s", + uuid, tableName, JSON.toJSON(bounds))); } } else if (obPartFuncType.isListPart()) { Map sets = parseFirstPartSets(rs, tableEntry); ((ObListPartDesc) tableEntry.getPartitionInfo().getFirstPartDesc()).setSets(sets); if (logger.isInfoEnabled()) { - logger.info(format( - "get first list sets from remote, sets=%s for tableEntry=%s", sets, - tableEntry)); + logger.info(format("uuid:%s, get first list sets from remote for %s, sets=%s", + uuid, tableName, JSON.toJSON(sets))); } + } else if (ObGlobal.obVsnMajor() >= 4 + && (obPartFuncType.isKeyPart() || obPartFuncType.isHashPart())) { + tableEntry.getPartitionInfo().setPartTabletIdMap( + parseFirstPartKeyHash(rs, tableEntry)); } } catch (Exception e) { - - logger.error(LCD.convert("01-00011"), tableEntry, obPartFuncType, e); + RUNTIME.error(LCD.convert("01-00011"), tableEntry, obPartFuncType, e); throw new ObTablePartitionInfoRefreshException(format( - "fail to get first part from remote, tableEntry=%s partFuncType=%s", tableEntry, - obPartFuncType), e); + "fail to get first part from remote for %s, tableEntry=%s partFuncType=%s", + tableName, tableEntry, obPartFuncType), e); } finally { try { if (null != rs) { @@ -544,32 +976,54 @@ private static void fetchFirstPart(Connection connection, TableEntry tableEntry, private static void fetchSubPart(Connection connection, TableEntry tableEntry, ObPartFuncType subPartFuncType) throws ObTablePartitionInfoRefreshException { + String tableName = ""; + TableEntryKey key = tableEntry.getTableEntryKey(); + if (key != null) { + tableName = key.getDatabaseName() + "." + key.getTableName(); + } + String uuid = UUID.randomUUID().toString(); + PreparedStatement pstmt = null; ResultSet rs = null; try { - pstmt = connection.prepareStatement(PROXY_SUB_PARTITION_SQL); - pstmt.setLong(1, tableEntry.getTableId()); - pstmt.setLong(2, TEMPLATE_PART_ID); - pstmt.setInt(3, Integer.MAX_VALUE); + if (ObGlobal.obVsnMajor() >= 4) { + pstmt = connection.prepareStatement(PROXY_SUB_PARTITION_SQL_V4); + pstmt.setString(1, key.getTenantName()); + pstmt.setLong(2, tableEntry.getTableId()); + pstmt.setInt(3, Integer.MAX_VALUE); + } else { + pstmt = connection.prepareStatement(PROXY_SUB_PARTITION_SQL); + pstmt.setLong(1, tableEntry.getTableId()); + pstmt.setInt(2, Integer.MAX_VALUE); + } rs = pstmt.executeQuery(); if (subPartFuncType.isRangePart()) { + List> highBoundVals = new ArrayList<>(); List> bounds = parseSubPartRange(rs, - tableEntry); + tableEntry, highBoundVals); ((ObRangePartDesc) tableEntry.getPartitionInfo().getSubPartDesc()) .setBounds(bounds); + ((ObRangePartDesc) tableEntry.getPartitionInfo().getSubPartDesc()) + .setHighBoundValues(highBoundVals); if (logger.isInfoEnabled()) { - logger.info(format("success to get sub ranges from remote, bounds=%s", bounds)); + logger.info(format("uuid:%s, get sub ranges from remote for %s, bounds=%s", + uuid, tableName, JSON.toJSON(bounds))); } } else if (subPartFuncType.isListPart()) { Map sets = parseSubPartSets(rs, tableEntry); ((ObListPartDesc) tableEntry.getPartitionInfo().getSubPartDesc()).setSets(sets); if (logger.isInfoEnabled()) { - logger.info(format("success to get sub list sets from remote, sets=%s", sets)); + logger.info(format("uuid:%s, get sub list sets from remote, sets=%s", uuid, + JSON.toJSON(sets))); } + } else if (ObGlobal.obVsnMajor() >= 4 + && (subPartFuncType.isKeyPart() || subPartFuncType.isHashPart())) { + tableEntry.getPartitionInfo().setPartTabletIdMap( + parseSubPartKeyHash(rs, tableEntry)); } } catch (Exception e) { - logger.error(LCD.convert("01-00012"), tableEntry, subPartFuncType, e); + RUNTIME.error(LCD.convert("01-00012"), tableEntry, subPartFuncType, e); throw new ObTablePartitionInfoRefreshException(format( "fail to get sub part from remote, tableEntry=%s partFuncType=%s", tableEntry, subPartFuncType), e); @@ -619,6 +1073,7 @@ private static TableEntry getTableEntryFromResultSet(TableEntryKey key, ResultSe entry.setPartitionNum(partitionNum); entry.setReplicaNum(replicaNum); } else { + RUNTIME.error("table not exist"); throw new ObTableNotExistException("table not exist: " + key.getTableName(), ResultCodes.OB_ERR_UNKNOWN_TABLE.errorCode); } @@ -627,17 +1082,40 @@ private static TableEntry getTableEntryFromResultSet(TableEntryKey key, ResultSe } private static ObPartitionEntry getPartitionLocationFromResultSet(TableEntry tableEntry, - ResultSet rs) - throws SQLException, - ObTablePartitionLocationRefreshException { - Map partitionLocation = new HashMap(); + ResultSet rs, + ObPartitionEntry partitionEntry) + throws SQLException, + ObTablePartitionLocationRefreshException { + if (partitionEntry == null || tableEntry == null) { + throw new IllegalArgumentException("partitionEntry: " + partitionEntry + + " tableEntry: " + tableEntry); + } + Map partitionLocation = partitionEntry.getPartitionLocation(); + Map tabletLsIdMap = partitionEntry.getTabletLsIdMap(); while (rs.next()) { ReplicaLocation replica = buildReplicaLocation(rs); - long partitionId = rs.getLong("partition_id"); + long partitionId; + if (ObGlobal.obVsnMajor() >= 4) { + partitionId = rs.getLong("tablet_id"); + long lsId = rs.getLong("ls_id"); + if (!rs.wasNull()) { + tabletLsIdMap.put(partitionId, lsId); + } else { + tabletLsIdMap.put(partitionId, INVALID_LS_ID); // non-partitioned table + } + } else { + partitionId = rs.getLong("partition_id"); + if (tableEntry.isPartitionTable() + && null != tableEntry.getPartitionInfo().getSubPartDesc()) { + partitionId = ObPartIdCalculator.getPartIdx(partitionId, tableEntry + .getPartitionInfo().getSubPartDesc().getPartNum()); + } + } if (!replica.isValid()) { - logger.warn(format( - "replica is invalid, continue, replica=%s, partitionId=%d, tableId=%d", - replica, partitionId, tableEntry.getTableId())); + RUNTIME + .warn(format( + "replica is invalid, continue, replica=%s, partitionId/tabletId=%d, tableId=%d", + replica, partitionId, tableEntry.getTableId())); continue; } ObPartitionLocation location = partitionLocation.get(partitionId); @@ -648,29 +1126,35 @@ private static ObPartitionEntry getPartitionLocationFromResultSet(TableEntry tab } location.addReplicaLocation(replica); } - ObPartitionEntry partitionEntry = new ObPartitionEntry(); - partitionEntry.setPartitionLocation(partitionLocation); - for (long i = 0; i < tableEntry.getPartitionNum(); i++) { - ObPartitionLocation location = partitionEntry.getPartitionLocationWithPartId(i); - if (location == null) { - logger.error(LCD.convert("01-00013"), i, partitionEntry, tableEntry); - throw new ObTablePartitionNotExistException(format( - "partition num=%d is not exist partitionEntry=%s original tableEntry=%s", i, - partitionEntry, tableEntry)); - } - if (location.getLeader() == null) { - logger.error(LCD.convert("01-00028"), i, partitionEntry, tableEntry); - throw new ObTablePartitionNoMasterException(format( - "partition num=%d has no leader partitionEntry=%s original tableEntry=%s", i, - partitionEntry, tableEntry)); + if (ObGlobal.obVsnMajor() < 4) { + for (long i = 0; i < tableEntry.getPartitionNum(); i++) { + ObPartitionLocation location = partitionEntry.getPartitionLocationWithPartId(i); + if (location == null) { + RUNTIME.error(LCD.convert("01-00013"), i, partitionEntry, tableEntry); + RUNTIME.error(format( + "partition num=%d is not exist partitionEntry=%s original tableEntry=%s", + i, partitionEntry, tableEntry)); + throw new ObTablePartitionNotExistException(format( + "partition num=%d is not exist partitionEntry=%s original tableEntry=%s", + i, partitionEntry, tableEntry)); + } + if (location.getLeader() == null) { + RUNTIME.error(LCD.convert("01-00028"), i, partitionEntry, tableEntry); + RUNTIME.error(format( + "partition num=%d has no leader partitionEntry=%s original tableEntry=%s", + i, partitionEntry, tableEntry)); + throw new ObTablePartitionNoMasterException(format( + "partition num=%d has no leader partitionEntry=%s original tableEntry=%s", + i, partitionEntry, tableEntry)); + } } } return partitionEntry; } - /** + /* * Get ReplicaLocation from the result row. * * @param rs @@ -709,19 +1193,27 @@ private static void fetchPartitionInfo(Connection connection, TableEntry tableEn ResultSet rs = null; ObPartitionInfo info = null; try { - pstmt = connection.prepareStatement(PROXY_PART_INFO_SQL); - pstmt.setLong(1, tableEntry.getTableId()); - pstmt.setLong(2, Long.MAX_VALUE); + if (ObGlobal.obVsnMajor() >= 4) { + pstmt = connection.prepareStatement(PROXY_PART_INFO_SQL_V4); + pstmt.setString(1, tableEntry.getTableEntryKey().getTenantName()); + pstmt.setLong(2, tableEntry.getTableId()); + pstmt.setLong(3, Long.MAX_VALUE); + } else { + pstmt = connection.prepareStatement(PROXY_PART_INFO_SQL); + pstmt.setLong(1, tableEntry.getTableId()); + pstmt.setLong(2, Long.MAX_VALUE); + } rs = pstmt.executeQuery(); info = parsePartitionInfo(rs); if (logger.isInfoEnabled()) { - logger.info("get part info from remote " + info); + logger.info("get part info from remote info:{}", JSON.toJSON(info)); } tableEntry.setPartitionInfo(info); } catch (Exception e) { - logger.error(LCD.convert("01-00014"), tableEntry); + RUNTIME.error(LCD.convert("01-00014"), tableEntry); + RUNTIME.error("fail to get part info from remote"); throw new ObTablePartitionInfoRefreshException(format( "fail to get part info from remote, tableEntry=%s", tableEntry), e); } finally { @@ -776,44 +1268,74 @@ private static ObPartitionInfo parsePartitionInfo(ResultSet rs) // get part key for each loop String partKeyExtra = rs.getString("part_key_extra"); partKeyExtra = partKeyExtra.replace("`", ""); // '`' is not supported by druid + partKeyExtra = partKeyExtra.replace(" ", ""); // ' ' should be removed ObColumn column; + String collationTypeLabel = null; + if (ObGlobal.obVsnMajor() >= 4) { + collationTypeLabel = "part_key_collation_type"; + } else { + collationTypeLabel = "spare1"; + } if (!partKeyExtra.isEmpty()) { column = new ObGeneratedColumn( rs.getString("part_key_name"),// rs.getInt("part_key_idx"),// ObObjType.valueOf(rs.getInt("part_key_type")),// - ObCollationType.valueOf(rs.getInt("spare1")), + ObCollationType.valueOf(rs.getInt(collationTypeLabel)), new ObGeneratedColumnExpressParser(getPlainString(partKeyExtra)).parse()); } else { column = new ObSimpleColumn(rs.getString("part_key_name"),// rs.getInt("part_key_idx"),// ObObjType.valueOf(rs.getInt("part_key_type")),// - ObCollationType.valueOf(rs.getInt("spare1"))); + ObCollationType.valueOf(rs.getInt(collationTypeLabel))); } info.addColumn(column); } // get list partition column types here - List orderedPartedColumns = null; + List orderedPartedColumns1 = null; if (null != info.getFirstPartDesc()) { if (info.getFirstPartDesc().getPartFuncType().isListPart() || info.getFirstPartDesc().getPartFuncType().isRangePart()) { - orderedPartedColumns = getOrderedPartColumns(info.getPartColumns(), + orderedPartedColumns1 = getOrderedPartColumns(info.getPartColumns(), info.getFirstPartDesc()); } } + + List orderedPartedColumns2 = null; if (null != info.getSubPartDesc()) { if (info.getSubPartDesc().getPartFuncType().isListPart() - || info.getFirstPartDesc().getPartFuncType().isRangePart()) { - orderedPartedColumns = getOrderedPartColumns(info.getPartColumns(), + || info.getSubPartDesc().getPartFuncType().isRangePart()) { + orderedPartedColumns2 = getOrderedPartColumns(info.getPartColumns(), info.getSubPartDesc()); } } // set the property of first part and sub part - setPartDescProperty(info.getFirstPartDesc(), info.getPartColumns(), orderedPartedColumns); - setPartDescProperty(info.getSubPartDesc(), info.getPartColumns(), orderedPartedColumns); + List firstPartColumns = new ArrayList(), subPartColumns = new ArrayList(); + if (null != info.getFirstPartDesc()) { + for (String partColumnNames : info.getFirstPartDesc().getOrderedPartColumnNames()) { + for (ObColumn curColumn : info.getPartColumns()) { + if (curColumn.getColumnName().equalsIgnoreCase(partColumnNames)) { + firstPartColumns.add(curColumn); + break; + } + } + } + } + if (null != info.getSubPartDesc()) { + for (String partColumnNames : info.getSubPartDesc().getOrderedPartColumnNames()) { + for (ObColumn curColumn : info.getPartColumns()) { + if (curColumn.getColumnName().equalsIgnoreCase(partColumnNames)) { + subPartColumns.add(curColumn); + break; + } + } + } + } + setPartDescProperty(info.getFirstPartDesc(), firstPartColumns, orderedPartedColumns1); + setPartDescProperty(info.getSubPartDesc(), subPartColumns, orderedPartedColumns2); return info; } @@ -831,6 +1353,8 @@ private static ObPartDesc buildPartDesc(ObPartitionLevel level, ResultSet rs) ObRangePartDesc rangeDesc = new ObRangePartDesc(); rangeDesc.setPartFuncType(partType); rangeDesc.setPartExpr(partExpr); + rangeDesc.setPartNum(rs.getInt(partLevelPrefix + "part_num")); + rangeDesc.setPartSpace(rs.getInt(partLevelPrefix + "part_space")); ArrayList types = new ArrayList(1); String objTypesStr = rs.getString(partLevelPrefix + "part_range_type"); for (String typeStr : objTypesStr.split(",")) { @@ -844,8 +1368,10 @@ private static ObPartDesc buildPartDesc(ObPartitionLevel level, ResultSet rs) hashDesc.setPartFuncType(partType); hashDesc.setPartNum(rs.getInt(partLevelPrefix + "part_num")); hashDesc.setPartSpace(rs.getInt(partLevelPrefix + "part_space")); - Map partNameIdMap = buildDefaultPartNameIdMap(hashDesc.getPartNum()); - hashDesc.setPartNameIdMap(partNameIdMap); + if (ObGlobal.obVsnMajor() < 4) { + Map partNameIdMap = buildDefaultPartNameIdMap(hashDesc.getPartNum()); + hashDesc.setPartNameIdMap(partNameIdMap); + } partDesc = hashDesc; } else if (partType.isKeyPart()) { ObKeyPartDesc keyPartDesc = new ObKeyPartDesc(); @@ -853,11 +1379,14 @@ private static ObPartDesc buildPartDesc(ObPartitionLevel level, ResultSet rs) keyPartDesc.setPartExpr(partExpr); keyPartDesc.setPartNum(rs.getInt(partLevelPrefix + "part_num")); keyPartDesc.setPartSpace(rs.getInt(partLevelPrefix + "part_space")); - Map partNameIdMap = buildDefaultPartNameIdMap(keyPartDesc.getPartNum()); - keyPartDesc.setPartNameIdMap(partNameIdMap); + if (ObGlobal.obVsnMajor() < 4) { + Map partNameIdMap = buildDefaultPartNameIdMap(keyPartDesc + .getPartNum()); + keyPartDesc.setPartNameIdMap(partNameIdMap); + } partDesc = keyPartDesc; } else { - logger.error(LCD.convert("01-00015"), partType); + RUNTIME.error(LCD.convert("01-00015"), partType); throw new IllegalArgumentException(format("not supported part type, type = %s", partType)); } @@ -886,6 +1415,7 @@ private static void setPartDescProperty(ObPartDesc partDesc, List part obPartFuncType = partDesc.getPartFuncType(); if (obPartFuncType.isKeyPart()) { if (partColumns == null || partColumns.size() == 0) { + RUNTIME.error("key part desc need part ref columns but found " + partColumns); throw new ObTablePartitionInfoRefreshException( "key part desc need part ref columns but found " + partColumns); } @@ -928,12 +1458,20 @@ private static Map buildPartNameIdMap(ObPartitionInfo partitionInf return partNameIdMap; } + private static Map parseFirstPartKeyHash(ResultSet rs, TableEntry tableEntry) + throws SQLException, + IllegalArgumentException, + FeatureNotSupportedException { + return parseKeyHashPart(rs, tableEntry, false); + } + private static List> parseFirstPartRange(ResultSet rs, - TableEntry tableEntry) - throws SQLException, - IllegalArgumentException, - FeatureNotSupportedException { - return parseRangePart(rs, tableEntry, false); + TableEntry tableEntry, + List> highBoundVals) + throws SQLException, + IllegalArgumentException, + FeatureNotSupportedException { + return parseRangePart(rs, tableEntry, highBoundVals, false); } private static Map parseFirstPartSets(ResultSet rs, TableEntry tableEntry) @@ -944,11 +1482,12 @@ private static Map parseFirstPartSets(ResultSet rs, TableE } private static List> parseSubPartRange(ResultSet rs, - TableEntry tableEntry) - throws SQLException, - IllegalArgumentException, - FeatureNotSupportedException { - return parseRangePart(rs, tableEntry, true); + TableEntry tableEntry, + List> highBoundVals) + throws SQLException, + IllegalArgumentException, + FeatureNotSupportedException { + return parseRangePart(rs, tableEntry, highBoundVals, true); } private static Map parseSubPartSets(ResultSet rs, TableEntry tableEntry) @@ -958,8 +1497,49 @@ private static Map parseSubPartSets(ResultSet rs, TableEnt return parseListPartSets(rs, tableEntry, true); } + private static Map parseSubPartKeyHash(ResultSet rs, TableEntry tableEntry) + throws SQLException, + IllegalArgumentException, + FeatureNotSupportedException { + return parseKeyHashPart(rs, tableEntry, true); + } + + private static Map parseKeyHashPart(ResultSet rs, TableEntry tableEntry, + boolean isSubPart) throws SQLException, + IllegalArgumentException, + FeatureNotSupportedException { + long idx = 0L; + Map partTabletIdMap = new HashMap(); + while (rs.next()) { + ObPartDesc subPartDesc = tableEntry.getPartitionInfo().getSubPartDesc(); + if (null != subPartDesc) { + // client only support template partition table + // so the sub_part_num is a constant and will store in subPartDesc which is different from proxy + if (subPartDesc instanceof ObKeyPartDesc) { + ObKeyPartDesc subKeyPartDesc = (ObKeyPartDesc) subPartDesc; + if (!isSubPart && subKeyPartDesc.getPartNum() == 0) { + long subPartNum = rs.getLong("sub_part_num"); + subKeyPartDesc.setPartNum((int) subPartNum); + } + } else if (subPartDesc instanceof ObHashPartDesc) { + ObHashPartDesc subHashPartDesc = (ObHashPartDesc) subPartDesc; + if (!isSubPart && subHashPartDesc.getPartNum() == 0) { + long subPartNum = rs.getLong("sub_part_num"); + subHashPartDesc.setPartNum((int) subPartNum); + } + } else { + throw new IllegalArgumentException("sub part desc is not key or hash part desc"); + } + } + Long tabletId = rs.getLong("tablet_id"); + partTabletIdMap.put(idx++, tabletId); + } + return partTabletIdMap; + } + private static List> parseRangePart(ResultSet rs, TableEntry tableEntry, + List> highBoundVals, boolean isSubPart) throws SQLException, IllegalArgumentException, @@ -974,38 +1554,94 @@ private static List> parseRangePart(ResultS List orderPartColumns = ((ObRangePartDesc) partDesc).getOrderedCompareColumns(); List> bounds = new ArrayList>(); Map partNameIdMap = new HashMap(); + Map partTabletIdMap = new HashMap(); + ObPartDesc subRangePartDesc = tableEntry.getPartitionInfo().getSubPartDesc(); + long idx = 0L; while (rs.next()) { + if (null != subRangePartDesc && !isSubPart && subRangePartDesc.getPartNum() == 0) { + // client only support template partition table + // so the sub_part_num is a constant and will store in subPartDesc which is different from proxy + long subPartNum = rs.getLong("sub_part_num"); + subRangePartDesc.setPartNum((int) subPartNum); + } + String highBoundVal = rs.getString("high_bound_val"); String[] splits = highBoundVal.split(","); List partElements = new ArrayList(); + List singleHighBoundVal = new ArrayList(); for (int i = 0; i < splits.length; i++) { String elementStr = getPlainString(splits[i]); if (elementStr.equalsIgnoreCase("MAXVALUE")) { + singleHighBoundVal.add(new byte[0]); // like EMPTY_BYTE_ARRAY partElements.add(MAX_PARTITION_ELEMENT); } else if (elementStr.equalsIgnoreCase("MINVALUE")) { + singleHighBoundVal.add(new byte[0]); // like EMPTY_BYTE_ARRAY partElements.add(MIN_PARTITION_ELEMENT); } else { - partElements - .add(orderPartColumns - .get(i) - .getObObjType() - .parseToComparable(elementStr, - orderPartColumns.get(i).getObCollationType())); + ObObjType type = orderPartColumns.get(i).getObObjType(); + partElements.add(type.parseToComparable(elementStr, orderPartColumns.get(i) + .getObCollationType())); + singleHighBoundVal.add(type.parseToBytes(elementStr, orderPartColumns.get(i) + .getObCollationType())); } } ObPartitionKey partitionKey = new ObPartitionKey(orderPartColumns, partElements); - Long partId = rs.getLong(partIdColumnName); - String partName = rs.getString("part_name"); - bounds.add(new ObComparableKV(partitionKey, partId)); - partNameIdMap.put(partName.toLowerCase(), partId); + if (ObGlobal.obVsnMajor() >= 4) { + long tabletId = rs.getLong("tablet_id"); + bounds.add(new ObComparableKV(partitionKey, idx)); + highBoundVals.add(singleHighBoundVal); + partTabletIdMap.put(idx, tabletId); + idx++; + } else { + long partId = rs.getLong(partIdColumnName); + String partName = rs.getString("part_name"); + bounds.add(new ObComparableKV(partitionKey, partId)); + highBoundVals.add(singleHighBoundVal); + partNameIdMap.put(partName.toLowerCase(), partId); + } + } + if (ObGlobal.obVsnMajor() >= 4) { + //set single level partition tablet-id mapping + tableEntry.getPartitionInfo().setPartTabletIdMap(partTabletIdMap); + } else { + //set single level partition name-id mapping + partDesc.setPartNameIdMap(partNameIdMap); } - //set single level partition name-id mapping - partDesc.setPartNameIdMap(partNameIdMap); Collections.sort(bounds); return bounds; } + private static String[] parseListPartSetsCommon(ResultSet rs, TableEntry tableEntry) + throws SQLException, + IllegalArgumentException, + FeatureNotSupportedException { + // multi-columns: '(1,2),(1,3),(1,4),(default)' + // single-columns: '(1),(2),(3)' or '1,2,3' + String setsStr = rs.getString("high_bound_val"); + setsStr = (null == setsStr) ? "" : setsStr.trim(); + if (setsStr.length() < 2) { + RUNTIME.error(LCD.convert("01-00016"), setsStr, tableEntry.toString()); + RUNTIME.error(format("high_bound_val value is error, high_bound_val=%s", setsStr)); + // if partition value format is wrong, directly throw exception + throw new IllegalArgumentException(format( + "high_bound_val value is error, high_bound_val=%s, tableEntry=%s", setsStr, + tableEntry.toString())); + } + // skip the first character '(' and the last ')' if exist + if (setsStr.startsWith("(") && setsStr.endsWith(")")) { + setsStr = setsStr.substring(1, setsStr.length() - 1); + } + + String[] setArray = null; + if (setsStr.contains("),(")) { // multi-column format + setArray = setsStr.split("\\),\\("); + } else { // single-column format + setArray = setsStr.split(","); + } + return setArray; + } + private static Map parseListPartSets(ResultSet rs, TableEntry tableEntry, boolean isSubPart) throws SQLException, @@ -1022,60 +1658,69 @@ private static Map parseListPartSets(ResultSet rs, TableEn List columns = ((ObListPartDesc) partDesc).getOrderCompareColumns(); Map sets = new HashMap(); - Map partNameIdMap = new HashMap(); - while (rs.next()) { - // multi-columns: '(1,2),(1,3),(1,4),(default)' - // single-columns: '(1),(2),(3)' or '1,2,3' - String setsStr = rs.getString("high_bound_val"); - setsStr = (null == setsStr) ? "" : setsStr.trim(); - if (setsStr.length() < 2) { - logger.error(LCD.convert("01-00016"), setsStr, tableEntry.toString()); - // if partition value format is wrong, directly throw exception - throw new IllegalArgumentException(format( - "high_bound_val value is error, high_bound_val=%s, tableEntry=%s", setsStr, - tableEntry.toString())); - } - // skip the first character '(' and the last ')' if exist - if (setsStr.startsWith("(") && setsStr.endsWith(")")) { - setsStr = setsStr.substring(1, setsStr.length() - 1); - } - - String[] setArray = null; - if (setsStr.contains("),(")) { // multi-column format - setArray = setsStr.split("\\),\\("); - } else { // single-column format - setArray = setsStr.split(","); - } - - ObPartitionKey key = null; - Long partId = null; - String partName = null; - // setArray can not be null - for (String set : setArray) { - if ("default".equalsIgnoreCase(set)) { - key = ObPartDesc.DEFAULT_PART_KEY; - } else { - String[] splits = set.split(","); + if (ObGlobal.obVsnMajor() >= 4) { + Map partTabletIdMap = new HashMap(); + long idx = 0L; + while (rs.next()) { + String[] setArray = parseListPartSetsCommon(rs, tableEntry); + ObPartitionKey key = null; + // setArray can not be null + for (String set : setArray) { + if ("default".equalsIgnoreCase(set)) { + key = ObPartDesc.DEFAULT_PART_KEY; + } else { + String[] splits = set.split(","); - List partElements = new ArrayList(); - for (int i = 0; i < splits.length; i++) { - partElements.add(columns.get(i).getObObjType() - .parseToComparable(splits[i], columns.get(i).getObCollationType())); + List partElements = new ArrayList(); + for (int i = 0; i < splits.length; i++) { + partElements.add(columns.get(i).getObObjType() + .parseToComparable(splits[i], columns.get(i).getObCollationType())); + } + key = new ObPartitionKey(columns, partElements); } - key = new ObPartitionKey(columns, partElements); + sets.put(key, idx); } - partId = rs.getLong(partIdColumnName); - partName = rs.getString("part_name"); - sets.put(key, partId); - partNameIdMap.put(partName.toLowerCase(), partId); + long tabletId = rs.getLong("tablet_id"); + partTabletIdMap.put(idx++, tabletId); } + //set single level partition tablet-id mapping + tableEntry.getPartitionInfo().setPartTabletIdMap(partTabletIdMap); + + } else { + Map partNameIdMap = new HashMap(); + while (rs.next()) { + String[] setArray = parseListPartSetsCommon(rs, tableEntry); + ObPartitionKey key = null; + Long partId = null; + String partName = null; + // setArray can not be null + for (String set : setArray) { + if ("default".equalsIgnoreCase(set)) { + key = ObPartDesc.DEFAULT_PART_KEY; + } else { + String[] splits = set.split(","); + + List partElements = new ArrayList(); + for (int i = 0; i < splits.length; i++) { + partElements.add(columns.get(i).getObObjType() + .parseToComparable(splits[i], columns.get(i).getObCollationType())); + } + key = new ObPartitionKey(columns, partElements); + } + partId = rs.getLong(partIdColumnName); + partName = rs.getString("part_name"); + sets.put(key, partId); + partNameIdMap.put(partName.toLowerCase(), partId); + } + } + //set single level partition name-id mapping + partDesc.setPartNameIdMap(partNameIdMap); } - //set single level partition name-id mapping - partDesc.setPartNameIdMap(partNameIdMap); + return sets; } - /** + /* * Load ocp model. */ public static OcpModel loadOcpModel(String paramURL, String dataSourceName, int connectTimeout, @@ -1105,6 +1750,8 @@ public static OcpModel loadOcpModel(String paramURL, String dataSourceName, int } if (obServerAddrs.isEmpty()) { + RUNTIME.error("load rs list failed dataSource: " + dataSourceName + " paramURL:" + + paramURL + " response:" + ocpResponse); throw new RuntimeException("load rs list failed dataSource: " + dataSourceName + " paramURL:" + paramURL + " response:" + ocpResponse); } @@ -1148,12 +1795,14 @@ private static OcpResponse getRemoteOcpResponseOrNull(String paramURL, String da } } catch (Exception e) { cause = e; - logger.error(LCD.convert("01-00017"), e); + RUNTIME.error(LCD.convert("01-00017"), e); Thread.sleep(retryInternal); } } if (tries >= tryTimes) { + RUNTIME + .error("Fail to get OCP response after " + tryTimes + " tries from [" + paramURL); throw new ObTableRetryExhaustedException("Fail to get OCP response after " + tryTimes + " tries from [" + paramURL + "], the content is [" + content + "]", cause); @@ -1161,7 +1810,7 @@ private static OcpResponse getRemoteOcpResponseOrNull(String paramURL, String da return null; } - /** + /* * Get IdcRegion info from OCP. * Return null instead of throwing exception if get nothing, because the info is optional. * @@ -1189,15 +1838,15 @@ private static OcpResponse getRemoteOcpIdcRegionOrNull(String paramURL, int conn return ocpResponse; } } catch (Exception e) { - logger.error(LCD.convert("01-00017"), e); + RUNTIME.error(LCD.convert("01-00017"), e); Thread.sleep(retryInternal); } } if (tries >= tryTimes) { - logger.error(LCD.convert("01-00017"), "OCP IdcRegion after" + tryTimes - + " tries from [" + paramURL - + "], the content is [" + content + "]"); + RUNTIME.error(LCD.convert("01-00017"), "OCP IdcRegion after" + tryTimes + + " tries from [" + paramURL + + "], the content is [" + content + "]"); } return null; } @@ -1231,7 +1880,7 @@ private static OcpResponse getLocalOcpResponseOrNull(String fileName) { } } } catch (IOException e) { - logger.warn("load obtable file meet exception: " + file.getAbsolutePath(), e); + RUNTIME.warn("load obtable file meet exception: " + file.getAbsolutePath(), e); return null; } @@ -1259,7 +1908,7 @@ private static void saveLocalContent(String fileName, String content) { } catch (IOException e) { // 写配置失败需要删除文件，避免写入脏数据 file.delete(); - logger.warn("Save obtable file meet exception: " + file.getAbsolutePath(), e); + RUNTIME.warn("Save obtable file meet exception: " + file.getAbsolutePath(), e); } } @@ -1300,4 +1949,34 @@ private static String getPlainString(String str) { .length(); return str.substring(start, end); } + + private static void parseObVersionFromSQL(String serverVersion) { + // serverVersion is like "4.2.1.0" + Pattern pattern = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)"); + Matcher matcher = pattern.matcher(serverVersion); + if (matcher.find() && ObGlobal.OB_VERSION == 0) { + ObGlobal.OB_VERSION = ObGlobal.calcVersion(Integer.parseInt(matcher.group(1)), + (short) Integer.parseInt(matcher.group(2)), + (byte) Integer.parseInt(matcher.group(3)), + (byte) Integer.parseInt(matcher.group(4))); + } + } + + public static void parseObVerionFromLogin(String serverVersion) { + Pattern pattern; + if (serverVersion.startsWith("OceanBase_CE")) { + // serverVersion in CE is like "OceanBase_CE 4.0.0.0" + pattern = Pattern.compile("OceanBase_CE\\s+(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)"); + } else { + // serverVersion is like "OceanBase 4.0.0.0" + pattern = Pattern.compile("OceanBase\\s+(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)"); + } + Matcher matcher = pattern.matcher(serverVersion); + if (matcher.find() && ObGlobal.OB_VERSION == 0) { + ObGlobal.OB_VERSION = ObGlobal.calcVersion(Integer.parseInt(matcher.group(1)), + (short) Integer.parseInt(matcher.group(2)), + (byte) Integer.parseInt(matcher.group(3)), + (byte) Integer.parseInt(matcher.group(4))); + } + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/SecureIdentityLoginModule.java b/src/main/java/com/alipay/oceanbase/rpc/location/SecureIdentityLoginModule.java index 19df9a5a..92ec4bdf 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/SecureIdentityLoginModule.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/SecureIdentityLoginModule.java @@ -30,7 +30,7 @@ public class SecureIdentityLoginModule { private static byte[] ENC_KEY_BYTES_PROD = "gQzLk5tTcGYlQ47GG29xQxfbHIURCheJ".getBytes(); - /** + /* * Decode. */ public static String decode(String secret) throws NoSuchPaddingException, @@ -39,7 +39,7 @@ public static String decode(String secret) throws NoSuchPaddingException, return decode(null, secret); } - /** + /* * Decode. */ public static String decode(String encKey, String secret) throws NoSuchPaddingException, diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObIndexInfo.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObIndexInfo.java new file mode 100644 index 00000000..e23652ac --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObIndexInfo.java @@ -0,0 +1,73 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.location.model; + +import com.alipay.oceanbase.rpc.protocol.payload.Constants; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObIndexType; + +public class ObIndexInfo { + private Long dataTableId = Constants.OB_INVALID_ID; + private Long indexTableId = Constants.OB_INVALID_ID; + private String indexTableName; + private ObIndexType indexType; + + public ObIndexInfo() { + this.dataTableId = Constants.OB_INVALID_ID; + this.indexTableId = Constants.OB_INVALID_ID; + this.indexTableName = ""; + this.indexType = ObIndexType.IndexTypeIsNot; + } + + public Long getDataTableId() { + return dataTableId; + } + + public void setDataTableId(Long dataTableId) { + this.dataTableId = dataTableId; + } + + public Long getIndexTableId() { + return indexTableId; + } + + public void setIndexTableId(Long indexTableId) { + this.indexTableId = indexTableId; + } + + public String getIndexTableName() { + return indexTableName; + } + + public void setIndexTableName(String indexTableName) { + this.indexTableName = indexTableName; + } + + public ObIndexType getIndexType() { + return indexType; + } + + public void setIndexType(ObIndexType indexType) { + this.indexType = indexType; + } + + @Override + public String toString() { + return "ObIndexInfo{" + "dataTableId=" + dataTableId + ", indexTableId=" + indexTableId + + ", indexTableName=" + indexTableName + ", indexType=" + indexType + '}'; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObReadConsistency.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObReadConsistency.java index 25248fd3..c7d19394 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObReadConsistency.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObReadConsistency.java @@ -39,28 +39,28 @@ public enum ObReadConsistency { } } - /** + /* * Value of. */ public static ObReadConsistency valueOf(int value) { return map.get(value); } - /** + /* * Get value. */ public int getValue() { return value; } - /** + /* * Get byte value. */ public byte getByteValue() { return (byte) value; } - /** + /* * Get ObReadConsistency by string value, return "STRONG" by default. */ static public ObReadConsistency getByName(String consistency) { @@ -70,7 +70,7 @@ static public ObReadConsistency getByName(String consistency) { return ObReadConsistency.STRONG; } - /** + /* * Is Strong consistency or not. * * @return @@ -79,7 +79,7 @@ public boolean isStrong() { return this.value == STRONG.value; } - /** + /* * Is weak consistency or not. * * @return @@ -88,7 +88,7 @@ public boolean isWeak() { return this.value == WEAK.value; } - /** + /* * Convert to ObTableConsistencyLevel. */ public ObTableConsistencyLevel toObTableConsistencyLevel() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObReplicaType.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObReplicaType.java index e1346220..72c8d8b6 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObReplicaType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObReplicaType.java @@ -43,7 +43,7 @@ public enum ObReplicaType { private String name; private int index; - /** + /* * Constructor. * * @param name @@ -54,7 +54,7 @@ public enum ObReplicaType { this.index = index; } - /** + /* * Get ReplicaType by idx. * * @param idx @@ -72,21 +72,21 @@ static public ObReplicaType getReplicaType(int idx) { } } - /** + /* * whether the replica is readable. */ public boolean isReadable() { return this.index == REPLICA_TYPE_FULL.index || this.index == REPLICA_TYPE_READONLY.index; } - /** + /* * whether the replica is readonly. */ public boolean isReadonly() { return this.index == REPLICA_TYPE_READONLY.index; } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObRoutePolicy.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObRoutePolicy.java index aa449364..43ec4fc0 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObRoutePolicy.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObRoutePolicy.java @@ -42,7 +42,7 @@ public enum ObRoutePolicy { private static Map map = new HashMap(); - /** + /* * Constructor. * * @param value @@ -57,7 +57,7 @@ public enum ObRoutePolicy { } } - /** + /* * Get ObRoutePolicy name, return "IDC_ORDER" by default. */ static public ObRoutePolicy getByName(String routePolicy) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerAddr.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerAddr.java index 14d175c6..d15ee269 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerAddr.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerAddr.java @@ -28,35 +28,35 @@ public class ObServerAddr implements Comparable { private volatile long grantPriorityTime = 0; private volatile long lastAccessTime = System.currentTimeMillis(); - /** + /* * Whether the addr is expired given the timeout. */ public boolean isExpired(long cachingTimeout) { return System.currentTimeMillis() - lastAccessTime > cachingTimeout; } - /** + /* * Record the access time. */ public void recordAccess() { lastAccessTime = System.currentTimeMillis(); } - /** + /* * Get ip. */ public String getIp() { return ip; } - /** + /* * Set ip. */ public void setIp(String ip) { this.ip = ip; } - /** + /* * Set address. */ public void setAddress(String address) { @@ -68,49 +68,49 @@ public void setAddress(String address) { } } - /** + /* * Get sql port. */ public int getSqlPort() { return sqlPort; } - /** + /* * Set sql port. */ public void setSqlPort(int sqlPort) { this.sqlPort = sqlPort; } - /** + /* * Get svr port. */ public int getSvrPort() { return svrPort; } - /** + /* * Set svr port. */ public void setSvrPort(int svrPort) { this.svrPort = svrPort; } - /** + /* * Get priority. */ public AtomicInteger getPriority() { return priority; } - /** + /* * Get grant priority time. */ public long getGrantPriorityTime() { return grantPriorityTime; } - /** + /* * Set grant priority time. */ public void setGrantPriorityTime(long grantPriorityTime) { @@ -121,7 +121,7 @@ public long getLastAccessTime() { return lastAccessTime; } - /** + /* * Equals. */ @Override @@ -134,7 +134,7 @@ public boolean equals(Object o) { return sqlPort == that.sqlPort && svrPort == that.svrPort && ip.equals(that.ip); } - /** + /* * Hash code. */ @Override @@ -142,7 +142,7 @@ public int hashCode() { return ip.hashCode() + sqlPort + svrPort; } - /** + /* * To string. */ @Override @@ -151,7 +151,7 @@ public String toString() { + svrPort + '}'; } - /** + /* * Compare to. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerInfo.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerInfo.java index 1e9599ee..6058287e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerInfo.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerInfo.java @@ -22,42 +22,42 @@ public class ObServerInfo { private long stopTime; private String status; // ACTIVE / INACTIVE / DELETING - /** + /* * Get status. */ public String getStatus() { return status; } - /** + /* * Set status. */ public void setStatus(String status) { this.status = status; } - /** + /* * Get stop time. */ public long getStopTime() { return stopTime; } - /** + /* * Set stop time. */ public void setStopTime(long stopTime) { this.stopTime = stopTime; } - /** + /* * Is active. */ public boolean isActive() { return stopTime == 0 && "active".equalsIgnoreCase(status); } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerLdcItem.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerLdcItem.java index 3bd5c247..8712b4f2 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerLdcItem.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerLdcItem.java @@ -27,7 +27,7 @@ public class ObServerLdcItem { private String idc; // physical idc private String region; // city - /** + /* * Constructor. * * @param ip @@ -42,35 +42,35 @@ public ObServerLdcItem(String ip, String zone, String idc, String region) { this.region = region; } - /** + /* * Get IP of the server. */ public String getIp() { return ip; } - /** + /* * Get Zone of the server. */ public String getZone() { return zone; } - /** + /* * Get IDC of the server. */ public String getIdc() { return idc; } - /** + /* * Get region of the server. */ public String getRegion() { return region; } - /** + /* * To String. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerLdcLocation.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerLdcLocation.java index 158722d1..100915dd 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerLdcLocation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerLdcLocation.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.Set; -/** +/* * ObServerLdcLocation organizes server by IDC->Region order. * */ @@ -44,7 +44,7 @@ enum RegionMatchType { private HashMap sameRegion = new HashMap(); private HashMap otherRegion = new HashMap(); - /** + /* * 构造 ObServerLdcLocation，根据当前 IDC 和 RegionMap 将服务器按 IDC->Region->OtherRegion的分类整理。 * * @param allServers @@ -103,49 +103,49 @@ public static ObServerLdcLocation buildLdcLocation(List allServ return loc; } - /** + /* * Whether the server in the same IDC. */ public boolean inSameIDC(String svr_ip) { return sameIDC.containsKey(svr_ip); } - /** + /* * Whether the server in the same region. */ public boolean inSameRegion(String ip) { return sameRegion.containsKey(ip); } - /** + /* * Whether the server in other region. */ public boolean inOtherRegion(String ip) { return otherRegion.containsKey(ip); } - /** + /* * Whether the LDC routing is enabled. */ public boolean isLdcUsed() { return isLdcUsed; } - /** + /* * Get current IDC. */ public String getCurrentIDC() { return currentIDC; } - /** + /* * Get Region Match Type. */ public RegionMatchType getMatchType() { return matchType; } - /** + /* * To String * * @return diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerRole.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerRole.java index ae247e7d..49b5d8c8 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerRole.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerRole.java @@ -48,14 +48,14 @@ static public ObServerRole getRole(String role) { } } - /** + /* * Get name. */ public String getName() { return name; } - /** + /* * Get index. */ public int getIndex() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerRoute.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerRoute.java index a7474bb2..f3bbc453 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerRoute.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObServerRoute.java @@ -34,14 +34,14 @@ public class ObServerRoute { // black ip list private Set blackIpList = new HashSet(); - /** + /* * Construct OB Server Route. */ private ObServerRoute(ObReadConsistency readConsistency) { this(readConsistency, ObRoutePolicy.IDC_ORDER, false); } - /** + /* * Construct OB Server Route. */ public ObServerRoute(ObReadConsistency readConsistency, ObRoutePolicy readRoutePolicy, @@ -52,49 +52,49 @@ public ObServerRoute(ObReadConsistency readConsistency, ObRoutePolicy readRouteP this.isLdcUsed = isLdcUsed; } - /** + /* * Get Consistency Level. */ public ObReadConsistency getReadConsistency() { return readConsistency; } - /** + /* * Get read route policy. */ public ObRoutePolicy getReadRoutePolicy() { return readRoutePolicy; } - /** + /* * Check whether LDC enabled. */ public boolean isLdcEnabled() { return isLdcUsed; } - /** + /* * Reset route, which will clean the black ip list. */ public void reset() { blackIpList.clear(); } - /** + /* * Check whether ip is in black list. */ public boolean isInBlackList(String ip) { return blackIpList.contains(ip); } - /** + /* * Add ip into black list. */ public void addToBlackList(String ip) { blackIpList.add(ip); } - /** + /* * set blackIpList. */ public void setBlackList(Set blackIpList) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObUserAuth.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObUserAuth.java index cdd76d53..45b48921 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ObUserAuth.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ObUserAuth.java @@ -30,13 +30,13 @@ public class ObUserAuth { private String encPassword; // encrypted password, set by user. - /** + /* * Default constructor. */ public ObUserAuth() { } - /** + /* * Constructor with user/password. */ public ObUserAuth(String userName, String password) { @@ -44,21 +44,21 @@ public ObUserAuth(String userName, String password) { this.password = password; } - /** + /* * Set User Name */ public void setUserName(String userName) { this.userName = userName; } - /** + /* * Set password */ public void setPassword(String password) { this.password = password; } - /** + /* * Set encrypted password. Decrypt it and set the password. */ public void setEncPassword(String encPassword) throws Exception { @@ -68,21 +68,21 @@ public void setEncPassword(String encPassword) throws Exception { } } - /** + /* * Get password. */ public String getPassword() { return password; } - /** + /* * Get user name. */ public String getUserName() { return userName; } - /** + /* * Get encrypted password set by user. *

* Attention: you won't get encrypted password if you haven't set it, even if you have set a password. diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpModel.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpModel.java index 9d8cfab8..a94e2841 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpModel.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpModel.java @@ -26,49 +26,53 @@ public class OcpModel { private long clusterId = -1; private HashMap idc2Region = new HashMap(); - /** + /* * Get ob server addrs. */ public List getObServerAddrs() { return obServerAddrs; } - /** + /* * Set ob server addrs. */ public void setObServerAddrs(List obServerAddrs) { this.obServerAddrs = obServerAddrs; } - /** + /* * Get cluster id. */ public long getClusterId() { return clusterId; } - /** + /* * Set cluster id. */ public void setClusterId(long clusterId) { this.clusterId = clusterId; } - /** + /* * Get Region by IDC. */ public String getIdc2Region(String idc) { return idc2Region.get(idc); } - /** + public HashMap getIdc2Region() { + return idc2Region; + } + + /* * Add Idc-Region pair. */ public void addIdc2Region(String idc, String region) { idc2Region.put(idc, region); } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponse.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponse.java index 1254f31b..fa20c1ba 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponse.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponse.java @@ -23,70 +23,70 @@ public class OcpResponse { private boolean Success; private OcpResponseData Data; - /** + /* * Get code. */ public int getCode() { return Code; } - /** + /* * Set code. */ public void setCode(int code) { Code = code; } - /** + /* * Get message. */ public String getMessage() { return Message; } - /** + /* * Set message. */ public void setMessage(String message) { Message = message; } - /** + /* * Is success. */ public boolean isSuccess() { return Success; } - /** + /* * Set success. */ public void setSuccess(boolean success) { Success = success; } - /** + /* * Get data. */ public OcpResponseData getData() { return Data; } - /** + /* * Validate. */ public boolean validate() { return isSuccess() && getCode() == 200 && Data != null && Data.validate(); } - /** + /* * Set data. */ public void setData(OcpResponseData data) { Data = data; } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseData.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseData.java index e7453a1a..e647b998 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseData.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseData.java @@ -25,70 +25,70 @@ public class OcpResponseData { private List RsList; private List IDCList; - /** + /* * Get ob region. */ public String getObRegion() { return ObRegion; } - /** + /* * Set ob region. */ public void setObRegion(String obRegion) { ObRegion = obRegion; } - /** + /* * Get ob region id. */ public long getObRegionId() { return ObRegionId; } - /** + /* * Set ob region id. */ public void setObRegionId(long obRegionId) { ObRegionId = obRegionId; } - /** + /* * Get rs list. */ public List getRsList() { return RsList; } - /** + /* * Set rs list. */ public void setRsList(List rsList) { RsList = rsList; } - /** + /* * Get IDC list. */ public List getIDCList() { return IDCList; } - /** + /* * Set IDC list. */ public void setIDCList(List IDCList) { this.IDCList = IDCList; } - /** + /* * Validate. */ public boolean validate() { return RsList != null && RsList.size() > 0; } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseDataIDC.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseDataIDC.java index eb200b59..bba403cc 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseDataIDC.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseDataIDC.java @@ -17,7 +17,7 @@ package com.alipay.oceanbase.rpc.location.model; -/** +/* * Idc->Region mapping defined in OCP. * */ @@ -26,28 +26,28 @@ public class OcpResponseDataIDC { private String idc; private String region; - /** + /* * Get idc. */ public String getIdc() { return idc; } - /** + /* * Set idc. */ public void setIdc(String idc) { this.idc = idc; } - /** + /* * Get region. */ public String getRegion() { return region; } - /** + /* * Set region. */ public void setRegion(String region) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseDataRs.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseDataRs.java index b202f0a8..43d8bf00 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseDataRs.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/OcpResponseDataRs.java @@ -22,49 +22,49 @@ public class OcpResponseDataRs { private String role; private int sql_port; - /** + /* * Get address. */ public String getAddress() { return address; } - /** + /* * Set address. */ public void setAddress(String address) { this.address = address; } - /** + /* * Get role. */ public String getRole() { return role; } - /** + /* * Set role. */ public void setRole(String role) { this.role = role; } - /** + /* * Get sql_port. */ public int getSql_port() { return sql_port; } - /** + /* * Set sql_port. */ public void setSql_port(int sql_port) { this.sql_port = sql_port; } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ReplicaLocation.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ReplicaLocation.java index df724c45..8936b029 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ReplicaLocation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ReplicaLocation.java @@ -24,7 +24,7 @@ public class ReplicaLocation { private ObServerRole role; private ObReplicaType replicaType; - /** + /* * Is valid. */ public boolean isValid() { @@ -32,91 +32,91 @@ public boolean isValid() { && (info.isActive()); } - /** + /* * Get addr. */ public ObServerAddr getAddr() { return addr; } - /** + /* * Set addr. */ public void setAddr(ObServerAddr addr) { this.addr = addr; } - /** + /* * Get role. */ public ObServerRole getRole() { return role; } - /** + /* * Set role. */ public void setRole(ObServerRole role) { this.role = role; } - /** + /* * Is leader. */ public boolean isLeader() { return ObServerRole.LEADER.equals(role); } - /** + /* * Get info. */ public ObServerInfo getInfo() { return info; } - /** + /* * Set info. */ public void setInfo(ObServerInfo info) { this.info = info; } - /** + /* * Get replicaType. */ public ObReplicaType getReplicaType() { return replicaType; } - /** + /* * Set replicaType. */ public void setReplicaType(ObReplicaType replicaType) { this.replicaType = replicaType; } - /** + /* * The replica is readable. */ public boolean isReadable() { return replicaType.isReadable(); } - /** + /* * The replica is readonly. */ public boolean isReadonly() { return replicaType.isReadonly(); } - /** + /* * Get Ip of the replica. */ public String getIp() { return addr.getIp(); } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/ServerRoster.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/ServerRoster.java index ab6adf84..870114ce 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/ServerRoster.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/ServerRoster.java @@ -35,7 +35,7 @@ public class ServerRoster { private AtomicReference serverLdc = new AtomicReference(); - /** + /* * Reset. */ public void reset(List members) { @@ -43,14 +43,14 @@ public void reset(List members) { this.roster.set(members); } - /** + /* * Reset Server LDC. */ public void resetServerLdc(ObServerLdcLocation serverLdc) { this.serverLdc.set(serverLdc); } - /** + /* * Upgrade max priority. */ public void upgradeMaxPriority(long priority) { @@ -67,7 +67,7 @@ public void upgradeMaxPriority(long priority) { resetMaxPriority(); } - /** + /* * Downgrade max priority. */ public void downgradeMaxPriority(long priority) { @@ -77,7 +77,7 @@ public void downgradeMaxPriority(long priority) { resetMaxPriority(); } - /** + /* * Reset max priority. */ public void resetMaxPriority() { @@ -94,28 +94,32 @@ public void resetMaxPriority() { } - /** + /* * Get max priority. */ public int getMaxPriority() { return maxPriority.get(); } - /** + /* * Get members. */ public List getMembers() { return roster.get(); } - /** + public AtomicReference> getRoster() { + return roster; + } + + /* * Get ServerLdcLocation. */ public ObServerLdcLocation getServerLdcLocation() { return serverLdc.get(); } - /** + /* * Choose a server by random. * * @param priorityTimeout @@ -140,7 +144,7 @@ public ObServerAddr getServer(long priorityTimeout, long cachingTimeout) { return addr; } - /** + /* * Reset priority and upgrade priority. * * @param addr @@ -156,7 +160,7 @@ public void resetPriority(ObServerAddr addr) { } } - /** + /* * Down grade priority and reorder the servers. * * @param addr @@ -169,7 +173,7 @@ public void downgradePriority(ObServerAddr addr) { } - /** + /* * To String. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java index c7b593be..559c9e04 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntry.java @@ -17,18 +17,21 @@ package com.alipay.oceanbase.rpc.location.model; +import com.alipay.oceanbase.rpc.location.model.partition.ObPartIdCalculator; import com.alipay.oceanbase.rpc.location.model.partition.ObPartitionEntry; import com.alipay.oceanbase.rpc.location.model.partition.ObPartitionInfo; +import com.alipay.oceanbase.rpc.location.model.partition.ObPartitionLevel; import com.alipay.oceanbase.rpc.protocol.payload.Constants; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import static com.google.common.base.Preconditions.checkArgument; public class TableEntry { - public static final Map HBASE_ROW_KEY_ELEMENT = new HashMap() { + public static final Map HBASE_ROW_KEY_ELEMENT = new LinkedHashMap() { { put("K", 0); put("Q", 1); @@ -42,6 +45,7 @@ public class TableEntry { private Long replicaNum = Constants.OB_INVALID_ID; private ObPartitionInfo partitionInfo = null; private volatile long refreshTimeMills; + private volatile long refreshAllTimeMills; private Map rowKeyElement = null; // table location @@ -50,7 +54,7 @@ public class TableEntry { private TableEntryKey tableEntryKey = null; private volatile ObPartitionEntry partitionEntry = null; - /** + /* * Is valid. */ public boolean isValid() { @@ -59,102 +63,116 @@ public boolean isValid() { } - /** + /* * Get table id. */ public Long getTableId() { return tableId; } - /** + /* * Set table id. */ public void setTableId(Long tableId) { this.tableId = tableId; } - /** + /* * Get partition num. */ public Long getPartitionNum() { return partitionNum; } - /** + /* * Set partition num. */ public void setPartitionNum(Long partitionNum) { this.partitionNum = partitionNum; } - /** + /* * Get replica num. */ public Long getReplicaNum() { return replicaNum; } - /** + /* * Is partition table. */ public boolean isPartitionTable() { return this.partitionNum > 1; } - /** + /* * Set replica num. */ public void setReplicaNum(Long replicaNum) { this.replicaNum = replicaNum; } - /** + /* * Get table location. */ public TableLocation getTableLocation() { return tableLocation; } - /** + /* * Set table location. */ public void setTableLocation(TableLocation tableLocation) { this.tableLocation = tableLocation; } - /** + /* * Get partition info. */ public ObPartitionInfo getPartitionInfo() { return partitionInfo; } - /** + /* * Set partition info. */ public void setPartitionInfo(ObPartitionInfo partitionInfo) { this.partitionInfo = partitionInfo; } - /** + /* * Get refresh time mills. */ public long getRefreshTimeMills() { return refreshTimeMills; } - /** + /* + * Get refresh time mills. + */ + public long getRefreshAllTimeMills() { + return refreshAllTimeMills; + } + + /* * Set refresh time mills. */ public void setRefreshTimeMills(long refreshTimeMills) { this.refreshTimeMills = refreshTimeMills; } + /* + * Set refresh all time mills. + */ + public void setRefreshAllTimeMills(long refreshAllTimeMills) { + this.refreshAllTimeMills = refreshAllTimeMills; + } + public Map getRowKeyElement() { return rowKeyElement; } - /** + /* * Set row key element. */ public void setRowKeyElement(Map rowKeyElement) { @@ -164,35 +182,35 @@ public void setRowKeyElement(Map rowKeyElement) { } } - /** + /* * Get table entry key. */ public TableEntryKey getTableEntryKey() { return tableEntryKey; } - /** + /* * Set table entry key. */ public void setTableEntryKey(TableEntryKey tableEntryKey) { this.tableEntryKey = tableEntryKey; } - /** + /* * Get partition entry. */ public ObPartitionEntry getPartitionEntry() { return partitionEntry; } - /** + /* * Set partition entry. */ public void setPartitionEntry(ObPartitionEntry partitionEntry) { this.partitionEntry = partitionEntry; } - /** + /* * Prepare. */ public void prepare() throws IllegalArgumentException { @@ -205,7 +223,20 @@ public void prepare() throws IllegalArgumentException { } } - /** + /* + * Get PartIdx from partId(logicId, partition id in 3.x) + */ + public long getPartIdx(long partId) { + long partIdx = partId; + if (this.getPartitionInfo() != null + && this.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_TWO) { + partIdx = ObPartIdCalculator.getPartIdx(partId, this.getPartitionInfo() + .getSubPartDesc().getPartNum()); + } + return partIdx; + } + + /* * Prepare for weak read. * @param ldcLocation */ @@ -215,15 +246,16 @@ public void prepareForWeakRead(ObServerLdcLocation ldcLocation) { } } - /** + /* * To string. */ @Override public String toString() { return "TableEntry{" + "tableId=" + tableId + ", partitionNum=" + partitionNum + ", replicaNum=" + replicaNum + ", partitionInfo=" + partitionInfo - + ", refreshTimeMills=" + refreshTimeMills + ", rowKeyElement=" + rowKeyElement - + ", tableLocation=" + tableLocation + ", tableEntryKey=" + tableEntryKey - + ", partitionEntry=" + partitionEntry + '}'; + + ", refreshTimeMills=" + refreshTimeMills + ", refreshAllTimeMills=" + + refreshAllTimeMills + ", rowKeyElement=" + rowKeyElement + ", tableLocation=" + + tableLocation + ", tableEntryKey=" + tableEntryKey + ", partitionEntry=" + + partitionEntry + '}'; } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntryKey.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntryKey.java index d266ad0e..03ac770d 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntryKey.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableEntryKey.java @@ -27,13 +27,13 @@ public class TableEntryKey { private String databaseName = Constants.EMPTY_STRING; private String tableName = Constants.EMPTY_STRING; - /** + /* * Table entry key. */ public TableEntryKey() { } - /** + /* * Table entry key. */ public TableEntryKey(String clusterName, String tenantName, String databaseName, @@ -45,7 +45,7 @@ public TableEntryKey(String clusterName, String tenantName, String databaseName, this.tableName = tableName; } - /** + /* * Get dummy entry key. */ public static TableEntryKey getDummyEntryKey(String clusterName, String tenantName) { @@ -54,21 +54,21 @@ public static TableEntryKey getDummyEntryKey(String clusterName, String tenantNa return dummyKey; } - /** + /* * Get sys dummy entry key. */ public static TableEntryKey getSysDummyEntryKey(String clusterName) { return getDummyEntryKey(clusterName, Constants.SYS_TENANT); } - /** + /* * Is sys all dummy. */ public boolean isSysAllDummy() { return this.tenantName.equals(Constants.SYS_TENANT) && this.isAllDummy(); } - /** + /* * Is all dummy. */ public boolean isAllDummy() { @@ -76,7 +76,7 @@ public boolean isAllDummy() { && this.tableName.equals(Constants.ALL_DUMMY_TABLE); } - /** + /* * Is valid. */ public boolean isValid() { @@ -84,7 +84,7 @@ public boolean isValid() { && StringUtil.isNotEmpty(this.databaseName) && StringUtil.isNotEmpty(this.tableName); } - /** + /* * To string. */ @Override @@ -93,7 +93,7 @@ public String toString() { + ", databaseName=" + databaseName + ", tableName=" + tableName + "]"; } - /** + /* * Hash code. */ @Override @@ -107,7 +107,7 @@ public int hashCode() { return result; } - /** + /* * Equals. */ @Override @@ -143,56 +143,56 @@ public boolean equals(Object obj) { return true; } - /** + /* * Get cluster name. */ public String getClusterName() { return clusterName; } - /** + /* * Set cluster name. */ public void setClusterName(String clusterName) { this.clusterName = clusterName; } - /** + /* * Get tenant name. */ public String getTenantName() { return tenantName; } - /** + /* * Set tenant name. */ public void setTenantName(String tenantName) { this.tenantName = tenantName; } - /** + /* * Get database name. */ public String getDatabaseName() { return databaseName; } - /** + /* * Set database name. */ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } - /** + /* * Get table name. */ public String getTableName() { return tableName; } - /** + /* * Set table name. */ public void setTableName(String tableName) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableLocation.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableLocation.java index 19e09f85..0a59cc2d 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/TableLocation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/TableLocation.java @@ -22,21 +22,21 @@ public class TableLocation { private List replicaLocations; - /** + /* * Get replica locations. */ public List getReplicaLocations() { return replicaLocations; } - /** + /* * Set replica locations. */ public void setReplicaLocations(List replicaLocations) { this.replicaLocations = replicaLocations; } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObComparableKV.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObComparableKV.java index b92deb4d..cf2ef96e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObComparableKV.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObComparableKV.java @@ -23,7 +23,7 @@ public class ObComparableKV, B> implements Comparable getPartIds(Object[] start, boolean startInclusive, Object[] end, + public List getPartIds(Object startRowObj, boolean startInclusive, Object endRowObj, boolean endInclusive) { // close set - try { - List startValues = evalRowKeyValues(start); + // verify the type of parameters and convert to Row + if (!(startRowObj instanceof Row) || !(endRowObj instanceof Row)) { + throw new ObTableException("invalid format of rowObj: " + startRowObj + ", " + + endRowObj); + } + Row startRow = (Row) startRowObj, endRow = (Row) endRowObj; + // pre-check start and end + // should remove after remove addRowkeyElement + if (startRow.size() != endRow.size()) { + throw new IllegalArgumentException("length of start key and end key is not equal"); + } + + if (startRow.size() == 1 && startRow.getValues()[0] instanceof ObObj && ((ObObj) startRow.getValues()[0]).isMinObj() && + endRow.size() == 1 && endRow.getValues()[0] instanceof ObObj && ((ObObj) endRow.getValues()[0]).isMaxObj()) { + return completeWorks; + } + + // check whether partition key is Min or Max, should refactor after remove addRowkeyElement + for (ObColumn curObcolumn : partColumns) { + for (int refIdx = 0; refIdx < curObcolumn.getRefColumnNames().size(); ++refIdx) { + String curObRefColumnName = curObcolumn.getRefColumnNames().get(refIdx); + if (startRow.size() <= refIdx) { + throw new IllegalArgumentException("rowkey length is " + startRow.size() + + ", which is shortest than " + refIdx); + } + // TODO: what if the curObRefColumnName does not exist in the startRow + if (startRow.get(curObRefColumnName) instanceof ObObj + && (((ObObj) startRow.get(curObRefColumnName)).isMinObj() || ((ObObj) startRow + .get(curObRefColumnName)).isMaxObj())) { + return completeWorks; + } + if (endRow.get(curObRefColumnName) instanceof ObObj + && (((ObObj) endRow.get(curObRefColumnName)).isMinObj() || ((ObObj) endRow + .get(curObRefColumnName)).isMaxObj())) { + return completeWorks; + } + } + } + + // eval partition key + List startValues = evalRowKeyValues(startRow); Object startValue = startValues.get(0); - List endValues = evalRowKeyValues(end); + List endValues = evalRowKeyValues(endRow); Object endValue = endValues.get(0); Long startLongValue = ObObjType.parseToLongOrNull(startValue); @@ -130,10 +175,9 @@ public List getPartIds(Object[] start, boolean startInclusive, Object[] en throw new IllegalArgumentException( "ObHashPartDesc get part id come across illegal params", e); } - } - /** + /* * Get random part id. */ @Override @@ -141,30 +185,34 @@ public Long getRandomPartId() { return ((this.partNum > 0) ? (long) RandomUtil.getRandomNum(0, this.partNum) : null); } - /** + /* * Get part id. */ @Override - public Long getPartId(Object... rowKey) { - List rowKeys = new ArrayList(); - rowKeys.add(rowKey); - return this.getPartId(rowKeys, false); + public Long getPartId(Object... row) { + List rows = new ArrayList(); + rows.addAll(Arrays.asList(row)); + return this.getPartId(rows, false); } - /** + /* * Get part id. */ @Override - public Long getPartId(List rowKeys, boolean consistency) { + public Long getPartId(List rows, boolean consistency) { - if (rowKeys == null || rowKeys.size() == 0) { - throw new IllegalArgumentException("invalid row keys :" + rowKeys); + if (rows == null || rows.size() == 0) { + throw new IllegalArgumentException("invalid row keys :" + rows); } Long partId = null; try { - for (Object[] rowKey : rowKeys) { - List evalValues = evalRowKeyValues(rowKey); + for (Object rowObj : rows) { + if (!(rowObj instanceof Row)) { + throw new ObTableException("invalid format of rowObj: " + rowObj); + } + Row row = (Row) rowObj; + List evalValues = evalRowKeyValues(row); Object value = evalValues.get(0);// the partition type of hash has one param at most Long longValue = ObObjType.parseToLongOrNull(value); @@ -183,9 +231,8 @@ public Long getPartId(List rowKeys, boolean consistency) { if (!partId.equals(currentPartId)) { throw new ObTablePartitionConsistentException( - "across partition operation may cause consistent problem " + rowKeys); + "across partition operation may cause consistent problem " + rows); } - } } catch (IllegalArgumentException e) { throw new IllegalArgumentException( @@ -196,10 +243,10 @@ public Long getPartId(List rowKeys, boolean consistency) { private Long innerHash(long hashValue) { hashValue = Math.abs(hashValue); - return (partSpace << ObPartConstants.PART_ID_BITNUM) | (hashValue % partNum); + return (partSpace << ObPartConstants.OB_PART_IDS_BITNUM) | (hashValue % partNum); } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java index 106e9d45..eb153770 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java @@ -17,16 +17,18 @@ package com.alipay.oceanbase.rpc.location.model.partition; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTablePartitionConsistentException; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObCollationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObColumn; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.util.ObHashUtils; import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; import org.apache.commons.lang.builder.ToStringBuilder; import org.slf4j.Logger; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.alipay.oceanbase.rpc.util.RandomUtil.getRandomNum; import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; @@ -37,60 +39,122 @@ */ public class ObKeyPartDesc extends ObPartDesc { private static final Logger logger = TableClientLoggerFactory.getLogger(ObKeyPartDesc.class); + private List completeWorks; private int partSpace = 0; private int partNum = 0; - /** + /* * Ob key part desc. */ public ObKeyPartDesc() { setPartFuncType(ObPartFuncType.KEY); } - /** + /* * Get part space. */ public int getPartSpace() { return partSpace; } - /** + /* * Set part space. */ public void setPartSpace(int partSpace) { this.partSpace = partSpace; } - /** + /* * Get part num. */ public int getPartNum() { return this.partNum; } - /** + /* * Set part num. */ public void setPartNum(int partNum) { this.partNum = partNum; + List partIds = new ArrayList(); + for (long i = 0; i < partNum; i++) { + partIds.add(i); + } + completeWorks = Collections.unmodifiableList(partIds); } - /** + /* * Get part ids. */ @Override - public List getPartIds(Object[] start, boolean startInclusive, Object[] end, + public List getPartIds(Object startRowObj, boolean startInclusive, Object endRowObj, boolean endInclusive) { - List rowKeys = new ArrayList(); - rowKeys.add(start); - rowKeys.add(end); - List partIds = new ArrayList(); - partIds.add(getPartId(rowKeys, true)); - return partIds; + try { + // verify the type of parameters and convert to Row + if (!(startRowObj instanceof Row) || !(endRowObj instanceof Row)) { + throw new ObTableException("invalid format of rowObj: " + startRowObj + ", " + + endRowObj); + } + Row startRow = (Row) startRowObj, endRow = (Row) endRowObj; + // pre-check start and end + // should remove after remove addRowkeyElement + if (startRow.size() != endRow.size()) { + throw new IllegalArgumentException("length of start key and end key is not equal"); + } + + if (startRow.size() == 1 && startRow.getValues()[0] instanceof ObObj && ((ObObj) startRow.getValues()[0]).isMinObj() && + endRow.size() == 1 && endRow.getValues()[0] instanceof ObObj && ((ObObj) endRow.getValues()[0]).isMaxObj()) { + return completeWorks; + } + + // check whether partition key is Min or Max, should refactor after remove addRowkeyElement + for (ObColumn curObcolumn : partColumns) { + for (int refIdx = 0; refIdx < curObcolumn.getRefColumnNames().size(); ++refIdx) { + String curObRefColumnName = curObcolumn.getRefColumnNames().get(refIdx); + if (startRow.size() <= refIdx) { + throw new IllegalArgumentException("rowkey length is " + startRow.size() + + ", which is shortest than " + refIdx); + } + if (startRow.get(curObRefColumnName) instanceof ObObj + && (((ObObj) startRow.get(curObRefColumnName)).isMinObj() || ((ObObj) startRow + .get(curObRefColumnName)).isMaxObj())) { + return completeWorks; + } + if (endRow.get(curObRefColumnName) instanceof ObObj + && (((ObObj) endRow.get(curObRefColumnName)).isMinObj() || ((ObObj) endRow + .get(curObRefColumnName)).isMaxObj())) { + return completeWorks; + } + } + } + + // eval partition key + List startValues = evalRowKeyValues(startRow); + List endValues = evalRowKeyValues(endRow); + + if (startValues == null || endValues == null) { + throw new NumberFormatException("can not parseToComparable start value [" + + startValues + "] or end value [" + endValues + + "] to long"); + } + + if (startValues.equals(endValues)) { + List partIds = new ArrayList(); + partIds.add(calcPartId(startValues)); + return partIds; + } else { + // partition key is different in key partition + return completeWorks; + } + } catch (IllegalArgumentException e) { + logger.error(LCD.convert("01-00002"), e); + throw new IllegalArgumentException( + "ObKeyPartDesc get part id come across illegal params", e); + } } - /** + /* * Get random part id. */ @Override @@ -98,32 +162,36 @@ public Long getRandomPartId() { return (this.partNum > 0 ? (long) getRandomNum(0, this.partNum) : null); } - /** + /* * Get part id. */ @Override - public Long getPartId(Object... rowKey) throws IllegalArgumentException { - List rowKeys = new ArrayList(); - rowKeys.add(rowKey); - return getPartId(rowKeys, false); + public Long getPartId(Object... row) throws IllegalArgumentException { + List rows = new ArrayList(); + rows.addAll(Arrays.asList(row)); + return getPartId(rows, false); } - /** + /* * Get part id. */ @Override - public Long getPartId(List rowKeys, boolean consistency) { + public Long getPartId(List rows, boolean consistency) { - if (rowKeys == null || rowKeys.size() == 0) { - throw new IllegalArgumentException("invalid row keys :" + rowKeys); + if (rows == null || rows.size() == 0) { + throw new IllegalArgumentException("invalid row keys :" + rows); } try { - int partRefColumnSize = orderedPartRefColumnRowKeyRelations.size(); + int partRefColumnSize = partColumns.size(); List evalValues = null; - for (Object[] rowKey : rowKeys) { - List currentRowKeyEvalValues = evalRowKeyValues(rowKey); + for (Object rowObj : rows) { + if (!(rowObj instanceof Row)) { + throw new ObTableException("invalid format of rowObj: " + rowObj); + } + Row row = (Row) rowObj; + List currentRowKeyEvalValues = evalRowKeyValues(row); if (evalValues == null) { evalValues = currentRowKeyEvalValues; } @@ -137,29 +205,38 @@ public Long getPartId(List rowKeys, boolean consistency) { } for (int i = 0; i < evalValues.size(); i++) { - if (!equalsWithCollationType(orderedPartRefColumnRowKeyRelations.get(i) - .getLeft().getObCollationType(), evalValues.get(i), - currentRowKeyEvalValues.get(i))) { + if (!equalsWithCollationType(partColumns.get(i).getObCollationType(), + evalValues.get(i), currentRowKeyEvalValues.get(i))) { throw new ObTablePartitionConsistentException( - "across partition operation may cause consistent problem " + rowKeys); + "across partition operation may cause consistent problem " + rows); } } } - Long hashValue = 0L; - for (int i = 0; i < partRefColumnSize; i++) { - hashValue = ObHashUtils.toHashcode(evalValues.get(i), - orderedPartRefColumnRowKeyRelations.get(i).getLeft(), hashValue); - } - - hashValue = (hashValue > 0 ? hashValue : -hashValue); - return (partSpace << ObPartConstants.PART_ID_BITNUM) | (hashValue % this.partNum); + return calcPartId(evalValues); } catch (IllegalArgumentException e) { logger.error(LCD.convert("01-00023"), e); throw new IllegalArgumentException( "ObKeyPartDesc get part id come across illegal params", e); } + } + + // calc partition id from eval values + private Long calcPartId(List evalValues) { + if (evalValues == null || evalValues.size() != partColumns.size()) { + throw new IllegalArgumentException("invalid eval values :" + evalValues); + } + + long hashValue = 0L; + for (int i = 0; i < partColumns.size(); i++) { + hashValue = ObHashUtils.toHashcode(evalValues.get(i), + partColumns.get(i), hashValue, + this.getPartFuncType()); + } + hashValue = (hashValue > 0 ? hashValue : -hashValue); + return ((long) partSpace << ObPartConstants.OB_PART_IDS_BITNUM) + | (hashValue % this.partNum); } private boolean equalsWithCollationType(ObCollationType collationType, Object s, Object t) @@ -175,7 +252,7 @@ private boolean equalsWithCollationType(ObCollationType collationType, Object s, } } - /** + /* * Set row key element. */ @Override @@ -186,7 +263,7 @@ public void setRowKeyElement(Map rowKeyElement) { super.setRowKeyElement(rowKeyElement); } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObListPartDesc.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObListPartDesc.java index a9acff96..3c3983f2 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObListPartDesc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObListPartDesc.java @@ -17,7 +17,9 @@ package com.alipay.oceanbase.rpc.location.model.partition; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTablePartitionConsistentException; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObColumn; import com.alipay.oceanbase.rpc.util.RandomUtil; import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; @@ -25,6 +27,7 @@ import org.slf4j.Logger; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -37,28 +40,28 @@ public class ObListPartDesc extends ObPartDesc { private List orderCompareColumns = null; private Map sets = null; - /** + /* * Ob list part desc. */ public ObListPartDesc() { this.setPartFuncType(ObPartFuncType.LIST); } - /** + /* * Get order compare columns. */ public List getOrderCompareColumns() { return this.orderCompareColumns; } - /** + /* * Set order compare columns. */ public void setOrderCompareColumns(List columns) { this.orderCompareColumns = columns; } - /** + /* * Set sets. */ public void setSets(Map sets) { @@ -69,7 +72,7 @@ public Map getSets() { return this.sets; } - /** + /* * Get random part id. */ @Override @@ -81,7 +84,7 @@ public Long getRandomPartId() { return (Long) this.sets.values().toArray()[randomIndex]; } - /** + /* * Prepare. */ @Override @@ -99,44 +102,41 @@ public void prepare() throws IllegalArgumentException { super.prepare(); } - /** + /* * Get part ids. */ @Override - public List getPartIds(Object[] start, boolean startInclusive, Object[] end, + public List getPartIds(Object startRowObj, boolean startInclusive, Object endRowObj, boolean endInclusive) { - List rowKeys = new ArrayList(); - rowKeys.add(start); - rowKeys.add(end); - Long partId = getPartId(rowKeys, true); - List partIds = new ArrayList(); - partIds.add(partId); - return partIds; + throw new IllegalArgumentException("getPartIds for List partition is not supported"); } - - /** + /* * Get part id. */ @Override - public Long getPartId(Object... rowKey) { - List rowKeys = new ArrayList(); - rowKeys.add(rowKey); - return getPartId(rowKeys, false); + public Long getPartId(Object... row) { + List rows = new ArrayList(); + rows.addAll(Arrays.asList(row)); + return getPartId(rows, false); } - /** + /* * Get part id. */ @Override - public Long getPartId(List rowKeys, boolean consistency) { - if (rowKeys == null || rowKeys.size() == 0) { - throw new IllegalArgumentException("invalid row keys :" + rowKeys); + public Long getPartId(List rows, boolean consistency) { + if (rows == null || rows.size() == 0) { + throw new IllegalArgumentException("invalid row keys :" + rows); } try { Long partId = null; - for (Object[] rowKey : rowKeys) { - List currentRowKeyEvalValues = evalRowKeyValues(rowKey); + for (Object rowObj : rows) { + if (!(rowObj instanceof Row)) { + throw new ObTableException("invalid format of rowObj: " + rowObj); + } + Row row = (Row) rowObj; + List currentRowKeyEvalValues = evalRowKeyValues(row); List values = super.initComparableElementByTypes( currentRowKeyEvalValues, this.orderCompareColumns); @@ -152,7 +152,7 @@ public Long getPartId(List rowKeys, boolean consistency) { if (partId != currentPartId) { throw new ObTablePartitionConsistentException( - "across partition operation may cause consistent problem " + rowKeys); + "across partition operation may cause consistent problem " + rows); } } @@ -165,7 +165,7 @@ public Long getPartId(List rowKeys, boolean consistency) { } - /** + /* * To string. */ @Override @@ -174,4 +174,8 @@ public String toString() { .append("sets", this.sets).append("partFuncType", this.getPartFuncType()) .append("partExpr", this.getPartExpr()).toString(); } + + @Override + public void setPartNum(int n) { + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPair.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPair.java index 6580ff7d..44b7b336 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPair.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPair.java @@ -21,7 +21,7 @@ public class ObPair { private final L left; private final R right; - /** + /* * Ob pair. */ public ObPair(L left, R right) { @@ -29,14 +29,14 @@ public ObPair(L left, R right) { this.right = right; } - /** + /* * Get left. */ public L getLeft() { return left; } - /** + /* * Get right. */ public R getRight() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartConstants.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartConstants.java index 2d51666f..78f4dfc7 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartConstants.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartConstants.java @@ -18,6 +18,6 @@ package com.alipay.oceanbase.rpc.location.model.partition; public interface ObPartConstants { - int PART_ID_BITNUM = 28; - int PART_ID_SHIFT = 32; + int OB_PART_IDS_BITNUM = 28; + int OB_PART_ID_SHIFT = 32; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartDesc.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartDesc.java index 8fdf1167..6be189ea 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartDesc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartDesc.java @@ -18,14 +18,19 @@ package com.alipay.oceanbase.rpc.location.model.partition; import com.alipay.oceanbase.rpc.exception.ObTablePartitionConsistentException; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObColumn; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.util.StringUtil; import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; +import com.alipay.oceanbase.rpc.mutation.Row; import org.slf4j.Logger; import java.util.*; import static com.alipay.oceanbase.rpc.constant.Constants.EMPTY_STRING; +import static com.alipay.oceanbase.rpc.location.model.partition.ObPartitionKey.MAX_PARTITION_ELEMENT; +import static com.alipay.oceanbase.rpc.location.model.partition.ObPartitionKey.MIN_PARTITION_ELEMENT; import static com.alipay.oceanbase.rpc.protocol.payload.impl.column.ObSimpleColumn.DEFAULT_UTF8MB4_GENERAL_CI; import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; import static java.util.Collections.*; @@ -51,53 +56,60 @@ public abstract class ObPartDesc { @SuppressWarnings("unchecked") protected Map rowKeyElement = EMPTY_MAP; - /** + /* * Get part func type. */ public ObPartFuncType getPartFuncType() { return partFuncType; } - /** + /* * Set part func type. */ public void setPartFuncType(ObPartFuncType partFuncType) { this.partFuncType = partFuncType; } - /** + /* * Get part expr. */ public String getPartExpr() { return partExpr; } - /** + /* * Set part expr. */ public void setPartExpr(String partExpr) { if (StringUtil.isBlank(partExpr)) { throw new IllegalArgumentException("ObKeyPartDesc part express is blank"); } - this.partExpr = partExpr; - this.orderedPartColumnNames = unmodifiableList(Arrays.asList(partExpr.split(","))); + this.partExpr = partExpr.replace(" ", ""); + this.orderedPartColumnNames = unmodifiableList(Arrays.asList(this.partExpr.split(","))); } - /** + /* + * Get part num + */ + public int getPartNum() { + return -1; + } + + /* * Get ordered part column names. */ public List getOrderedPartColumnNames() { return orderedPartColumnNames; } - /** + /* * Get part columns. */ public List getPartColumns() { return partColumns; } - /** + /* * Set part columns. */ public void setPartColumns(List partColumns) { @@ -108,7 +120,7 @@ public Map getPartNameIdMap() { return this.partNameIdMap; } - /** + /* * Set part name id map. */ public void setPartNameIdMap(Map partNameIdMap) { @@ -119,7 +131,7 @@ public Map getRowKeyElement() { return rowKeyElement; } - /** + /* * Set row key element. */ public void setRowKeyElement(Map rowKeyElement) { @@ -132,8 +144,21 @@ protected List initComparableElementByTypes(List objects, try { for (int i = 0; i < objects.size(); i++) { ObColumn obColumn = obColumns.get(i); - comparableElement.add(obColumn.getObObjType().parseToComparable(objects.get(i), - obColumn.getObCollationType())); + if (objects.get(i) instanceof ObObj) { + // deal with min / max + ObObj obj = (ObObj) objects.get(i); + if (obj.isMinObj()) { + comparableElement.add(MIN_PARTITION_ELEMENT); + } else if (obj.isMaxObj()) { + comparableElement.add(MAX_PARTITION_ELEMENT); + } else { + throw new IllegalArgumentException(String.format( + "failed to cast obj, obj=%s, types=%s", objects, obColumns)); + } + } else { + comparableElement.add(obColumn.getObObjType().parseToComparable(objects.get(i), + obColumn.getObCollationType())); + } } } catch (Exception e) { logger.error(LCD.convert("01-00024"), objects, obColumns, e); @@ -145,92 +170,72 @@ protected List initComparableElementByTypes(List objects, //to prepare partition calculate resource //to check partition calculate is ready - public void prepare() throws IllegalArgumentException { - if (orderedPartColumnNames == EMPTY_LIST) { - throw new IllegalArgumentException( - "prepare ObPartDesc failed. orderedPartColumnNames is empty"); - } + public void prepare() throws IllegalArgumentException { /* do nothing now */ } - if (rowKeyElement == null || rowKeyElement.size() == 0) { - throw new IllegalArgumentException("prepare ObPartDesc failed. rowKeyElement is empty"); + /* + * Eval row key values. + */ + public List evalRowKeyValues(Row row) throws IllegalArgumentException { + int partColumnSize = partColumns.size(); + List evalValues = new ArrayList(partColumnSize); + Object[] rowValues = row.getValues(); + String[] rowColumnNames = row.getColumns(); + + if (rowValues.length < partColumnSize) { + throw new IllegalArgumentException("Input row key should at least include " + partColumns + + "but found" + Arrays.toString(rowValues)); } - if (partColumns == null || partColumns.size() == 0) { - throw new IllegalArgumentException("prepare ObPartDesc failed. partColumns is empty"); - } - List>> orderPartRefColumnRowKeyRelations = new ArrayList>>( - orderedPartColumnNames.size()); - for (String partOrderColumnName : orderedPartColumnNames) { - for (ObColumn column : partColumns) { - if (column.getColumnName().equalsIgnoreCase(partOrderColumnName)) { - List partRefColumnRowKeyIndexes = new ArrayList(column - .getRefColumnNames().size()); - for (String refColumn : column.getRefColumnNames()) { - boolean rowKeyElementRefer = false; - for (String rowKeyElementName : rowKeyElement.keySet()) { - if (rowKeyElementName.equalsIgnoreCase(refColumn)) { - partRefColumnRowKeyIndexes - .add(rowKeyElement.get(rowKeyElementName)); - rowKeyElementRefer = true; + + boolean needEval = true; + + // column or generate column + for (int i = 0; i < partColumns.size(); ++i) { + ObColumn curObColumn = partColumns.get(i); + List curObRefColumnNames = curObColumn.getRefColumnNames(); + Object[] evalParams = new Object[curObRefColumnNames.size()]; + for (int j = 0; j < curObRefColumnNames.size(); ++j) { + for (int k = 0; k < rowColumnNames.length; ++k) { + if (rowColumnNames[k].equalsIgnoreCase(curObRefColumnNames.get(j))) { + if (curObRefColumnNames.size() == 1 && rowValues[k] instanceof ObObj) { + ObObj obj = (ObObj) rowValues[k]; + if (obj.isMaxObj() || obj.isMinObj()) { + evalValues.add(obj); + needEval = false; + break; + } } - } - if (!rowKeyElementRefer) { - throw new IllegalArgumentException("partition order column " - + partOrderColumnName - + " refer to non-row-key column " - + refColumn); + evalParams[j] = rowValues[k]; + break; } } - orderPartRefColumnRowKeyRelations.add(new ObPair>( - column, partRefColumnRowKeyIndexes)); + } + if (needEval) { + evalValues.add(curObColumn.evalValue(evalParams)); } } - } - this.orderedPartRefColumnRowKeyRelations = orderPartRefColumnRowKeyRelations; - } - - /** - * Eval row key values. - */ - public List evalRowKeyValues(Object... rowKey) throws IllegalArgumentException { - int partRefColumnSize = orderedPartRefColumnRowKeyRelations.size(); - List evalValues = new ArrayList(partRefColumnSize); - // column or generate column - for (int i = 0; i < partRefColumnSize; i++) { - ObPair> orderedPartRefColumnRowKeyRelation = orderedPartRefColumnRowKeyRelations - .get(i); - if (rowKey.length != rowKeyElement.size()) { - throw new IllegalArgumentException("row key is consist of " + rowKeyElement - + "but found" + Arrays.toString(rowKey)); - } - // row key is consists of multi column - List refIndex = orderedPartRefColumnRowKeyRelation.getRight(); - Object[] evalParams = new Object[refIndex.size()]; - for (int j = 0; j < refIndex.size(); j++) { - //TODO where get the type of ref column ? - evalParams[j] = rowKey[refIndex.get(j)]; - } - evalValues.add(orderedPartRefColumnRowKeyRelation.getLeft().evalValue(evalParams)); - } return evalValues; } - /** + /* * * @param start the start row key * @param startInclusive the start row key inclusive * @param end the end row key * @param endInclusive the end row key inclusive */ - public abstract List getPartIds(Object[] start, boolean startInclusive, Object[] end, - boolean endInclusive) throws IllegalArgumentException; + public abstract List getPartIds(Object startRowObj, boolean startInclusive, + Object endRowObj, boolean endInclusive) + throws IllegalArgumentException; - public abstract Long getPartId(Object... rowKey) throws IllegalArgumentException; + public abstract Long getPartId(Object... row) throws IllegalArgumentException; - public abstract Long getPartId(List rowKeys, boolean consistency) - throws IllegalArgumentException, - ObTablePartitionConsistentException; + public abstract Long getPartId(List row, boolean consistency) + throws IllegalArgumentException, + ObTablePartitionConsistentException; public abstract Long getRandomPartId(); + + public abstract void setPartNum(int n); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartFuncType.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartFuncType.java index d34e2758..f1441882 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartFuncType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartFuncType.java @@ -26,7 +26,8 @@ public enum ObPartFuncType { KEY_V2("KEY_V2", 6), // LIST_COLUMNS("LIST_COLUMNS", 7), // HASH_V2("HASH_V2", 8), // - KEY_V3("KEY_V3", 9); + KEY_V3("KEY_V3", 9), // + KEY_IMPLICIT_V2("KEY_IMPLICIT_V2", 10); private final String name; private final long index; @@ -36,7 +37,7 @@ public enum ObPartFuncType { this.index = index; } - /** + /* * Get ob part func type. */ public static ObPartFuncType getObPartFuncType(long index) { @@ -60,12 +61,14 @@ public static ObPartFuncType getObPartFuncType(long index) { return HASH_V2; } else if (KEY_V3.index == index) { return KEY_V3; + } else if (KEY_IMPLICIT_V2.index == index) { + return KEY_IMPLICIT_V2; } else { return UNKNOWN; } } - /** + /* * Get ob part func type. */ public static ObPartFuncType getObPartFuncType(String name) { @@ -89,48 +92,51 @@ public static ObPartFuncType getObPartFuncType(String name) { return HASH_V2; } else if (KEY_V3.name.equalsIgnoreCase(name)) { return KEY_V3; + } else if (KEY_IMPLICIT_V2.name.equalsIgnoreCase(name)) { + return KEY_IMPLICIT_V2; } else { return UNKNOWN; } } - /** + /* * Get index. */ public long getIndex() { return index; } - /** + /* * Get name. */ public String getName() { return name; } - /** + /* * Is range part. */ public boolean isRangePart() { return this.index == RANGE.getIndex() || this.index == RANGE_COLUMNS.getIndex(); } - /** + /* * Is hash part. */ public boolean isHashPart() { return this.index == HASH.getIndex() || this.index == HASH_V2.getIndex(); } - /** + /* * Is key part. */ public boolean isKeyPart() { - return this.index == KEY_IMPLICIT.getIndex() || this.index == KEY_V2.getIndex() - || this.index == KEY_V3.getIndex(); + return this.index == KEY_IMPLICIT_V2.getIndex() || this.index == KEY_IMPLICIT.getIndex() + || this.index == KEY_V2.getIndex() || this.index == KEY_V3.getIndex() + || this.index == KEY.getIndex(); } - /** + /* * Is list part. */ public boolean isListPart() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartIdCalculator.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartIdCalculator.java index 76060daf..969642a9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartIdCalculator.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartIdCalculator.java @@ -17,14 +17,22 @@ package com.alipay.oceanbase.rpc.location.model.partition; -import static com.alipay.oceanbase.rpc.location.model.partition.ObPartConstants.PART_ID_BITNUM; -import static com.alipay.oceanbase.rpc.location.model.partition.ObPartConstants.PART_ID_SHIFT; +import static com.alipay.oceanbase.rpc.location.model.partition.ObPartConstants.OB_PART_IDS_BITNUM; +import static com.alipay.oceanbase.rpc.location.model.partition.ObPartConstants.OB_PART_ID_SHIFT; public class ObPartIdCalculator { - private static final Long MASK = (1L << PART_ID_BITNUM) - | 1L << (PART_ID_BITNUM + PART_ID_SHIFT); + public static final Long OB_TWOPART_BEGIN_MASK = (1L << OB_PART_IDS_BITNUM) + | 1L << (OB_PART_IDS_BITNUM + OB_PART_ID_SHIFT); - /** + /* + * Get PartIdx (logicId) from partId. + */ + public static long getPartIdx(long partId, int subPartNum) { + // template partition id is generated by first_part_id and sub_part_id one by one + return extractPartIdx(partId) * subPartNum + extractSubpartIdx(partId); + } + + /* * Generate part id. */ public static Long generatePartId(Long partId1, Long partId2) { @@ -37,7 +45,36 @@ public static Long generatePartId(Long partId1, Long partId2) { return partId1; } - return (partId1 << PART_ID_SHIFT) | partId2 | MASK; + return (partId1 << OB_PART_ID_SHIFT) | partId2 | OB_TWOPART_BEGIN_MASK; + } + + // get subpart_id with PARTITION_LEVEL_TWO_MASK + public static long extractSubpartId(long id) { + return id & (~(0xFFFFFFFFFFFFFFFFL << OB_PART_ID_SHIFT)); + } + + // get part_id with PARTITION_LEVEL_TWO_MASK + public static long extractPartId(long id) { + return id >> OB_PART_ID_SHIFT; } + // get part idx from one level partid + public static long extractIdxFromPartId(long id) { + return id & (~(0xFFFFFFFFFFFFFFFFL << OB_PART_IDS_BITNUM)); + } + + // get part space from one level partid + public static long extractSpaceFromPartId(long id) { + return id >> OB_PART_IDS_BITNUM; + } + + // get part_idx + public static long extractPartIdx(long id) { + return extractIdxFromPartId(extractPartId(id)); + } + + // get sub_part_idx + public static long extractSubpartIdx(long id) { + return extractIdxFromPartId(extractSubpartId(id)); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java index 41d1dfd7..d3e87e71 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionEntry.java @@ -25,25 +25,28 @@ public class ObPartitionEntry { private Map partitionLocation = new HashMap(); + // mapping from tablet id to ls id, and the part id to tablet id mapping is in ObPartitionInfo + private Map tabletLsIdMap = new HashMap<>(); + public Map getPartitionLocation() { return partitionLocation; } - /** + /* * Set partition location. */ public void setPartitionLocation(Map partitionLocation) { this.partitionLocation = partitionLocation; } - /** + /* * Get partition location with part id. */ public ObPartitionLocation getPartitionLocationWithPartId(long partId) { return partitionLocation.get(partId); } - /** + /* * Put partition location with part id. */ public ObPartitionLocation putPartitionLocationWithPartId(long partId, @@ -51,7 +54,22 @@ public ObPartitionLocation putPartitionLocationWithPartId(long partId, return partitionLocation.put(partId, ObpartitionLocation); } - /** + /* + * Get partition location with tablet id. + */ + public ObPartitionLocation getPartitionLocationWithTabletId(long tabletId) { + return partitionLocation.get(tabletId); + } + + /* + * Put partition location with part id. + */ + public ObPartitionLocation putPartitionLocationWithTabletId(long tabletId, + ObPartitionLocation ObpartitionLocation) { + return partitionLocation.put(tabletId, ObpartitionLocation); + } + + /* * Prepare for weak read. * @param ldcLocation */ @@ -61,11 +79,21 @@ public void prepareForWeakRead(ObServerLdcLocation ldcLocation) { } } - /** + /* * To string. */ @Override public String toString() { return "ObPartitionEntry{" + "partitionLocation=" + partitionLocation + '}'; } + + public Map getTabletLsIdMap() { + return tabletLsIdMap; + } + + public void setTabletLsIdMap(Map tabletLsIdMap) { + this.tabletLsIdMap = tabletLsIdMap; + } + + public long getLsId(long tabletId) { return tabletLsIdMap.get(tabletId); } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionInfo.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionInfo.java index a94469e8..b6d9e083 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionInfo.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionInfo.java @@ -26,63 +26,65 @@ import static com.google.common.base.Preconditions.checkArgument; public class ObPartitionInfo { - private ObPartitionLevel level = ObPartitionLevel.UNKNOWN; - private ObPartDesc firstPartDesc = null; - private ObPartDesc subPartDesc = null; - private List partColumns = new ArrayList(1); - private Map partNameIdMap = null; - private Map rowKeyElement = null; - - /** + private ObPartitionLevel level = ObPartitionLevel.UNKNOWN; + private ObPartDesc firstPartDesc = null; + private ObPartDesc subPartDesc = null; + private List partColumns = new ArrayList(1); + // mapping from part id to tablet id, and the tablet id to ls id mapping is in ObPartitionInfo + private Map partTabletIdMap = null; + private Map partNameIdMap = null; + private Map rowKeyElement = null; + + /* * Get level. */ public ObPartitionLevel getLevel() { return level; } - /** + /* * Set level. */ public void setLevel(ObPartitionLevel level) { this.level = level; } - /** + /* * Get first part desc. */ public ObPartDesc getFirstPartDesc() { return firstPartDesc; } - /** + /* * Set first part desc. */ public void setFirstPartDesc(ObPartDesc firstPartDesc) { this.firstPartDesc = firstPartDesc; } - /** + /* * Get sub part desc. */ public ObPartDesc getSubPartDesc() { return subPartDesc; } - /** + /* * Set sub part desc. */ public void setSubPartDesc(ObPartDesc subPartDesc) { this.subPartDesc = subPartDesc; } - /** + /* * Get part columns. */ public List getPartColumns() { return partColumns; } - /** + /* * Add column. */ public void addColumn(ObColumn column) { @@ -93,7 +95,7 @@ public Map getRowKeyElement() { return rowKeyElement; } - /** + /* * Set row key element. */ public void setRowKeyElement(Map rowKeyElement) { @@ -106,7 +108,7 @@ public void setRowKeyElement(Map rowKeyElement) { } } - /** + /* * Prepare. */ public void prepare() throws IllegalArgumentException { @@ -126,7 +128,7 @@ public void prepare() throws IllegalArgumentException { } } - /** + /* * To string. */ @Override @@ -144,10 +146,21 @@ public Map getPartNameIdMap() { return this.partNameIdMap; } - /** + /* * Set part name id map. */ public void setPartNameIdMap(Map partNameIdMap) { this.partNameIdMap = partNameIdMap; } + + public Map getPartTabletIdMap() { + return this.partTabletIdMap; + } + + /* + * Set part tablet id map. + */ + public void setPartTabletIdMap(Map partTabletIdMap) { + this.partTabletIdMap = partTabletIdMap; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionKey.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionKey.java index 23ccc702..7f9c9dca 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionKey.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionKey.java @@ -26,7 +26,7 @@ public class ObPartitionKey implements Comparable { public static final Comparable MAX_PARTITION_ELEMENT = new Comparable() { - /** + /* * Compare to. */ @Override @@ -39,7 +39,7 @@ public int compareTo(Object o) { }; public static final Comparable MIN_PARTITION_ELEMENT = new Comparable() { - /** + /* * Compare to. */ @Override @@ -54,7 +54,15 @@ public int compareTo(Object o) { private final List partitionElements; private final List orderPartColumns; - /** + public List getPartitionElements() { + return partitionElements; + } + + public List getOrderPartColumns() { + return orderPartColumns; + } + + /* * Ob partition key. */ public ObPartitionKey(List orderPartColumns, List partitionElements) @@ -73,7 +81,7 @@ public ObPartitionKey(List orderPartColumns, List partitio @Override @SuppressWarnings("unchecked") - /** + /* * Compare to. */ public int compareTo(ObPartitionKey that) { @@ -107,6 +115,11 @@ public int compareTo(ObPartitionKey that) { tmpRet = ((String) thisElement).toUpperCase().compareTo( ((String) thatElement).toUpperCase()); } else { + // make number into long value for compare + if (thisElement instanceof Number && thatElement instanceof Number) { + thisElement = ((Number) thisElement).longValue(); + thatElement = ((Number) thatElement).longValue(); + } tmpRet = thisElement.compareTo(thatElement); } @@ -118,7 +131,7 @@ public int compareTo(ObPartitionKey that) { return 0; } - /** + /* * Get instance. */ public static ObPartitionKey getInstance(List orderPartColumns, @@ -126,7 +139,7 @@ public static ObPartitionKey getInstance(List orderPartColumns, return new ObPartitionKey(orderPartColumns, Arrays.asList(rowKeyElements)); } - /** + /* * Get instance. */ public static ObPartitionKey getInstance(List orderPartColumns, @@ -134,7 +147,7 @@ public static ObPartitionKey getInstance(List orderPartColumns, return new ObPartitionKey(orderPartColumns, rowKeyElements); } - /** + /* * Equals. */ @Override @@ -181,7 +194,7 @@ public boolean equals(Object o) { return true; } - /** + /* * Hash code. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLevel.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLevel.java index 298b3db4..3dcc23ea 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLevel.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLevel.java @@ -31,7 +31,7 @@ public enum ObPartitionLevel { this.index = index; } - /** + /* * Value of. */ public static ObPartitionLevel valueOf(long index) { @@ -47,14 +47,14 @@ public static ObPartitionLevel valueOf(long index) { } } - /** + /* * Get index. */ public long getIndex() { return index; } - /** + /* * Get name. */ public String getName() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocation.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocation.java index 448cba4d..9b4421ca 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObPartitionLocation.java @@ -33,20 +33,30 @@ public class ObPartitionLocation { private List sameRegion = new ArrayList(); private List otherRegion = new ArrayList(); - /** + /* * Get leader. */ public ReplicaLocation getLeader() { - if (leader == null) { - // Previously, exception is thrown when get table meta with any no leader partition, - // thus might prevent us from updating other partitions. - // Now, put off the exception until we need to access the leader server instead. - throw new ObTablePartitionNoMasterException("partition has no leader."); - } return leader; } - /** + public List getReplicas() { + return replicas; + } + + public List getSameIdc() { + return sameIdc; + } + + public List getSameRegion() { + return sameRegion; + } + + public List getOtherRegion() { + return otherRegion; + } + + /* * Add replication * * @param replica @@ -58,7 +68,7 @@ public void addReplicaLocation(ReplicaLocation replica) { this.replicas.add(replica); } - /** + /* * Get replica according to route strategy. * * @param route @@ -78,7 +88,7 @@ public ReplicaLocation getReplica(ObServerRoute route) { } } - /** + /* * Get read replica according to LDC route strategy. * * @param route @@ -108,7 +118,7 @@ public ReplicaLocation getReadReplicaByLDC(ObServerRoute route) { return leader; } - /** + /* * Get read replica according to LDC route strategy. * * @param route @@ -124,7 +134,7 @@ public ReplicaLocation getReadReplicaNoLdc(ObServerRoute route) { return leader; } - /** + /* * Classify Replica for weak read, according to Server LDC location. * Synchronized to avoid duplicate initialization. * @@ -145,7 +155,7 @@ public void prepareForWeakRead(ObServerLdcLocation ldcLocation) { } } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDesc.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDesc.java index 9e3799a9..b41871e9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDesc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDesc.java @@ -17,8 +17,11 @@ package com.alipay.oceanbase.rpc.location.model.partition; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTablePartitionConsistentException; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObColumn; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; import com.alipay.oceanbase.rpc.protocol.payload.impl.column.ObGeneratedColumn; import com.alipay.oceanbase.rpc.protocol.payload.impl.column.ObSimpleColumn; @@ -29,32 +32,43 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; +import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.RUNTIME; public class ObRangePartDesc extends ObPartDesc { private static final Logger logger = TableClientLoggerFactory.getLogger(ObRangePartDesc.class); - /** + /* * Ob range part desc. */ public ObRangePartDesc() { setPartFuncType(ObPartFuncType.RANGE); } + private int partSpace = 0; + private int partNum = 0; private List orderedCompareColumns = null; private List orderedCompareColumnTypes = null; private List> bounds = null; - - /** + // save the upper bound of the partition key + // demo: + // PARTITION P0 VALUES LESS THAN (500, 't', 't'), + // PARTITION P0 VALUES LESS THAN (1000, 'T', 'T'), + // -> [[500, t, t], [1000, T, T]] + private List> highBoundValues = null; + + /* * Get ordered compare column types. */ public List getOrderedCompareColumnTypes() { return orderedCompareColumnTypes; } + private List completeWorks; - /** + /* * Set ordered compare column types. */ public void setOrderedCompareColumnTypes(List orderedPartColumnTypes) { @@ -62,21 +76,21 @@ public void setOrderedCompareColumnTypes(List orderedPartColumnTypes) this.orderedCompareColumnTypes = orderedPartColumnTypes; } - /** + /* * Set ordered compare columns. */ public void setOrderedCompareColumns(List orderedPartColumn) { this.orderedCompareColumns = orderedPartColumn; } - /** + /* * Get ordered compare columns. */ public List getOrderedCompareColumns() { return orderedCompareColumns; } - /** + /* * Set bounds. */ public void setBounds(List> bounds) { @@ -87,7 +101,18 @@ public List> getBounds() { return bounds; } - /** + /* + * Set highBoundValues + */ + public void setHighBoundValues(List> highBoundValues) { + this.highBoundValues = highBoundValues; + } + + public List> getHighBoundValues() { + return highBoundValues; + } + + /* * Get random part id. */ @Override @@ -99,7 +124,7 @@ public Long getRandomPartId() { return this.bounds.get(randomIndex).value; } - /** + /* * Prepare. */ @Override @@ -152,63 +177,168 @@ public void prepare() throws IllegalArgumentException { super.prepare(); } - /** + /* + * Get part space. + */ + public int getPartSpace() { + return partSpace; + } + + /* + * Set part space. + */ + public void setPartSpace(int partSpace) { + this.partSpace = partSpace; + } + + /* + * Get part num. + */ + public int getPartNum() { + return this.partNum; + } + + /* + * Set part num. + */ + public void setPartNum(int partNum) { + this.partNum = partNum; + List partIds = new ArrayList(); + for (long i = 0; i < partNum; i++) { + partIds.add(i); + } + completeWorks = Collections.unmodifiableList(partIds); + } + + /* * Get part ids. */ @Override - public List getPartIds(Object[] start, boolean startInclusive, Object[] end, + public List getPartIds(Object startRowObj, boolean startInclusive, Object endRowObj, boolean endInclusive) { - // can not detail the border effect so that the range is magnified - long startPartId = getPartId(start); - long stopPartId = getPartId(end); + if (!(startRowObj instanceof Row) || !(endRowObj instanceof Row)) { + throw new ObTableException("invalid format of rowObj: " + startRowObj + ", " + + endRowObj); + } + Row startRow = (Row) startRowObj, endRow = (Row) endRowObj; + // pre-check start and end + // should remove after remove addRowkeyElement + if (startRow.size() != endRow.size()) { + throw new IllegalArgumentException("length of start key and end key is not equal"); + } + if (startRow.size() == 1 && startRow.getValues()[0] instanceof ObObj && ((ObObj) startRow.getValues()[0]).isMinObj() && + endRow.size() == 1 && endRow.getValues()[0] instanceof ObObj && ((ObObj) endRow.getValues()[0]).isMaxObj()) { + return completeWorks; + } + + int startIdx = getBoundsIdx(true, startRow); + int stopIdx = getBoundsIdx(true, endRow); List partIds = new ArrayList(); - for (long i = startPartId; i <= stopPartId; i++) { - partIds.add(i); + for (int i = startIdx; i <= stopIdx; i++) { + partIds.add(this.bounds.get(i).value); } return partIds; } - /** + /* * Get part id. */ @Override - public Long getPartId(Object... rowKey) { + public Long getPartId(Object... row) { + try { + List rows = new ArrayList(); + rows.addAll((Arrays.asList(row))); + return this.bounds.get(getBoundsIdx(false, rows)).value; + } catch (IllegalArgumentException e) { + RUNTIME.error(LCD.convert("01-00025"), e); + throw new IllegalArgumentException( + "ObRangePartDesc get part id come across illegal params", e); + } + + } - if (rowKey.length != rowKeyElement.size()) { - throw new IllegalArgumentException("row key is consist of " + rowKeyElement - + "but found" + Arrays.toString(rowKey)); + public int getBoundsIdx(boolean isScan, List rowObj) { + if (!(rowObj.get(0) instanceof Row)) { + throw new ObTableException("invalid format of rowObj: " + rowObj); + } + Row row = (Row) rowObj.get(0); + if (row.size() < partColumns.size()) { + throw new IllegalArgumentException("Input row key should at least include " + partColumns + + "but found" + Arrays.toString(row.getValues())); } try { - List evalParams = evalRowKeyValues(rowKey); + List evalParams = evalRowKeyValues(row); List comparableElement = super.initComparableElementByTypes(evalParams, this.orderedCompareColumns); ObPartitionKey searchKey = ObPartitionKey.getInstance(orderedCompareColumns, comparableElement); - int idx = upperBound(this.bounds, new ObComparableKV(searchKey, + + int pos = upperBound(this.bounds, new ObComparableKV(searchKey, (long) -1)); - return this.bounds.get(idx).value; + if (pos >= this.bounds.size()) { + if (isScan) { + // if range is bigger than rangeMax while scanning + // we just scan until last range + return this.bounds.size() - 1; + } + throw new ArrayIndexOutOfBoundsException("Table has no partition for value in " + + this.getPartExpr()); + } else { + return pos; + } } catch (IllegalArgumentException e) { - logger.error(LCD.convert("01-00025"), e); - throw new IllegalArgumentException( - "ObRangePartDesc get part id come across illegal params", e); + RUNTIME.error(LCD.convert("01-00025"), e); + throw new IllegalArgumentException("ObRangePartDesc get getBoundsIdx error", e); } } - /** + public int getBoundsIdx(boolean isScan, Row rowKey) { + try { + List evalParams = evalRowKeyValues(rowKey); + List comparableElement = super.initComparableElementByTypes(evalParams, + this.orderedCompareColumns); + ObPartitionKey searchKey = ObPartitionKey.getInstance(orderedCompareColumns, + comparableElement); + + int pos = upperBound(this.bounds, new ObComparableKV(searchKey, + (long) -1)); + if (pos >= this.bounds.size()) { + if (isScan) { + // if range is bigger than rangeMax while scanning + // we just scan until last range + return this.bounds.size() - 1; + } + throw new ArrayIndexOutOfBoundsException("Table has no partition for value in " + + this.getPartExpr()); + } else { + return pos; + } + } catch (IllegalArgumentException e) { + RUNTIME.error(LCD.convert("01-00025"), e); + throw new IllegalArgumentException("ObRangePartDesc get getBoundsIdx error", e); + } + + } + + /* * Get part id. */ @Override - public Long getPartId(List rowKeys, boolean consistency) { - if (rowKeys == null || rowKeys.size() == 0) { - throw new IllegalArgumentException("invalid row keys :" + rowKeys); + public Long getPartId(List rows, boolean consistency) { + if (rows == null || rows.size() == 0) { + throw new IllegalArgumentException("invalid row keys :" + rows); } Long partId = null; - for (Object[] rowKey : rowKeys) { - long currentPartId = getPartId(rowKey); + for (Object rowObj : rows) { + if (!(rowObj instanceof Row)) { + throw new ObTableException("invalid format of rowObj: " + rowObj); + } + Row row = (Row) rowObj; + long currentPartId = getPartId(row); if (partId == null) { partId = currentPartId; } @@ -218,7 +348,7 @@ public Long getPartId(List rowKeys, boolean consistency) { if (!partId.equals(currentPartId)) { throw new ObTablePartitionConsistentException( - "across partition operation may cause consistent problem " + rowKeys); + "across partition operation may cause consistent problem " + rows); } } @@ -227,7 +357,7 @@ public Long getPartId(List rowKeys, boolean consistency) { private static > int upperBound(List list, T key) { int first = 0; - int len = list.size() - 1; + int len = list.size(); int half = 0; int middle = 0; while (len > 0) { @@ -245,7 +375,7 @@ private static > int upperBound(List list, T return first; } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Append.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Append.java new file mode 100644 index 00000000..4549c9a0 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Append.java @@ -0,0 +1,147 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.table.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Append extends Mutation { + private List columns; + private List values; + boolean withResult; + + /* + * default constructor + */ + public Append() { + super(); + columns = new ArrayList(); + values = new ArrayList(); + withResult = false; + } + + /* + * construct with ObTableClient and String + */ + public Append(Table client, String tableName) { + super(client, tableName); + columns = new ArrayList(); + values = new ArrayList(); + withResult = false; + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return ObTableOperationType.APPEND; + } + + /* + * get the mutated columns' name + */ + public String[] getColumns() { + return columns.toArray(new String[0]); + } + + /* + * get the mutated columns' value + */ + public Object[] getValues() { + return values.toArray(); + } + + /* + * add mutated Row + */ + public Append addMutateRow(Row rows) { + if (null == rows) { + throw new IllegalArgumentException("Invalid null rowKey set into Update"); + } else if (0 == rows.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set mutate row into Update + for (Map.Entry entry : rows.getMap().entrySet()) { + columns.add(entry.getKey()); + values.add(entry.getValue()); + } + + return this; + } + + /* + * add mutated ColumnValues + */ + public Append addMutateColVal(ColumnValue... columnValues) { + if (null == columnValues) { + throw new IllegalArgumentException("Invalid null columnValues set into Update"); + } + + // set mutate ColumnValue into Update + for (ColumnValue columnValue : columnValues) { + if (columns.contains(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in Row Key"); + } + columns.add(columnValue.getColumnName()); + values.add(columnValue.getValue()); + } + + return this; + } + + /* + * Remove rowkey from mutateColval + */ + public Append removeRowkeyFromMutateColval() { + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + return this; + } + + /* + * execute + */ + public MutationResult execute() throws Exception { + if (null == getTableName() || getTableName().isEmpty()) { + throw new ObTableException("table name is null"); + } else if (null == getClient()) { + throw new ObTableException("client is null"); + } + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + if (null == getQuery()) { + // simple append, without filter + return new MutationResult(((ObTableClient) getClient()).appendWithResult( + getTableName(), getRowKey(), getKeyRanges(), columns.toArray(new String[0]), + values.toArray(), withResult)); + } else { + // QueryAndAppend + ObTableOperation operation = ObTableOperation.getInstance(ObTableOperationType.APPEND, + new Object[] {}, columns.toArray(new String[0]), values.toArray()); + return new MutationResult(((ObTableClient) getClient()).mutationWithFilter(getQuery(), + getRowKey(), getKeyRanges(), operation, withResult)); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java new file mode 100644 index 00000000..6caf1887 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/BatchOperation.java @@ -0,0 +1,320 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.checkandmutate.CheckAndInsUp; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.table.ObTableClientLSBatchOpsImpl; +import com.alipay.oceanbase.rpc.table.api.Table; +import com.alipay.oceanbase.rpc.table.api.TableBatchOps; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.ObGlobal; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class BatchOperation { + private String tableName; + private Table client; + boolean withResult; + private List operations; + boolean isAtomic = false; + boolean returnOneResult = false; + boolean hasCheckAndInsUp = false; + boolean hasGet = false; + ObTableOperationType lastType = ObTableOperationType.INVALID; + boolean isSameType = true; + protected ObTableEntityType entityType = ObTableEntityType.KV; + + /* + * default constructor + */ + public BatchOperation() { + tableName = null; + client = null; + withResult = false; + operations = new ArrayList<>(); + } + + /* + * construct with client and table name + */ + public BatchOperation(Table client, String tableName) { + this.tableName = tableName; + this.client = client; + withResult = false; + operations = new ArrayList<>(); + } + + /* + * set client + */ + public BatchOperation setClient(Table client) { + this.client = client; + return this; + } + + /* + * set table + */ + public BatchOperation setTable(String tableName) { + this.tableName = tableName; + return this; + } + + /* + * add queries + */ + public BatchOperation addOperation(TableQuery... queries) { + if (isSameType && lastType != ObTableOperationType.INVALID + && lastType != ObTableOperationType.GET) { + isSameType = false; + } + + lastType = ObTableOperationType.GET; + this.operations.addAll(Arrays.asList(queries)); + return this; + } + + /* + * add mutations + */ + public BatchOperation addOperation(Mutation... mutations) { + for (int i = 0; i < mutations.length; i++) { + if (isSameType && lastType != ObTableOperationType.INVALID + && lastType != mutations[i].getOperationType()) { + isSameType = false; + } + lastType = mutations[i].getOperationType(); + } + this.operations.addAll(Arrays.asList(mutations)); + return this; + } + + /* + * add mutations + */ + public BatchOperation addOperation(List mutations) { + for (int i = 0; i < mutations.size(); i++) { + if (isSameType && lastType != ObTableOperationType.INVALID + && lastType != mutations.get(i).getOperationType()) { + isSameType = false; + } + lastType = mutations.get(i).getOperationType(); + } + this.operations.addAll(mutations); + return this; + } + + /* + * add CheckAndInsUp + */ + public BatchOperation addOperation(CheckAndInsUp... insUps) { + if (isSameType && lastType != ObTableOperationType.INVALID + && lastType != ObTableOperationType.CHECK_AND_INSERT_UP) { + isSameType = false; + } + lastType = ObTableOperationType.CHECK_AND_INSERT_UP; + this.operations.addAll(Arrays.asList(insUps)); + this.hasCheckAndInsUp = true; + return this; + } + + public void setEntityType(ObTableEntityType entityType) { + this.entityType = entityType; + } + + public BatchOperation setIsAtomic(boolean isAtomic) { + this.isAtomic = isAtomic; + return this; + } + + public BatchOperation setReturnOneResult(boolean returnOneResult) { + this.returnOneResult = returnOneResult; + return this; + } + + @SuppressWarnings("unchecked") + public BatchOperationResult execute() throws Exception { + if (returnOneResult + && !(isSameType && (lastType == ObTableOperationType.INSERT + || lastType == ObTableOperationType.PUT + || lastType == ObTableOperationType.REPLACE || lastType == ObTableOperationType.DEL))) { + throw new IllegalArgumentException( + "returnOneResult only support multi-insert/put/replace/del"); + } + + if (hasCheckAndInsUp || ObGlobal.isLsOpSupport()) { + return executeWithLSBatchOp(); + } else { + return executeWithNormalBatchOp(); + } + } + + private BatchOperationResult executeWithNormalBatchOp() throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + TableBatchOps batchOps = client.batch(tableName); + boolean hasSetRowkeyElement = false; + + for (Object operation : operations) { + if (operation instanceof Mutation) { + Mutation mutation = (Mutation) operation; + if (!hasSetRowkeyElement && mutation.getRowKeyNames() != null) { + List rowKeyNames = mutation.getRowKeyNames(); + ((ObTableClient) client).addRowKeyElement(tableName, + rowKeyNames.toArray(new String[0])); + hasSetRowkeyElement = true; + } + ObTableOperationType type = mutation.getOperationType(); + switch (type) { + case GET: + throw new IllegalArgumentException("Invalid type in batch operation, " + + type); + case INSERT: + ((Insert) mutation).removeRowkeyFromMutateColval(); + batchOps.insert(((Insert) mutation).getRowKeyValues().toArray(new Object[0]), ((Insert) mutation).getColumns(), + ((Insert) mutation).getValues()); + break; + case DEL: + batchOps.delete(((Delete) mutation).getRowKeyValues().toArray(new Object[0])); + break; + case UPDATE: + ((Update) mutation).removeRowkeyFromMutateColval(); + batchOps.update(((Update) mutation).getRowKeyValues().toArray(new Object[0]), ((Update) mutation).getColumns(), + ((Update) mutation).getValues()); + break; + case INSERT_OR_UPDATE: + ((InsertOrUpdate) mutation).removeRowkeyFromMutateColval(); + batchOps.insertOrUpdate(((InsertOrUpdate) mutation).getRowKeyValues().toArray(new Object[0]), + ((InsertOrUpdate) mutation).getColumns(), + ((InsertOrUpdate) mutation).getValues()); + break; + case REPLACE: + ((Replace) mutation).removeRowkeyFromMutateColval(); + batchOps.replace(((Replace) mutation).getRowKeyValues().toArray(new Object[0]), ((Replace) mutation).getColumns(), + ((Replace) mutation).getValues()); + break; + case INCREMENT: + ((Increment) mutation).removeRowkeyFromMutateColval(); + batchOps.increment(((Increment) mutation).getRowKeyValues().toArray(new Object[0]), + ((Increment) mutation).getColumns(), + ((Increment) mutation).getValues(), withResult); + break; + case APPEND: + ((Append) mutation).removeRowkeyFromMutateColval(); + batchOps.append(((Append) mutation).getRowKeyValues().toArray(new Object[0]), ((Append) mutation).getColumns(), + ((Append) mutation).getValues(), withResult); + break; + case PUT: + ((Put) mutation).removeRowkeyFromMutateColval(); + batchOps.put(((Put) mutation).getRowKeyValues().toArray(new Object[0]), ((Put) mutation).getColumns(), + ((Put) mutation).getValues()); + break; + default: + throw new ObTableException("unknown operation type " + type); + } + } else if (operation instanceof TableQuery) { + TableQuery query = (TableQuery) operation; + batchOps.get(query.getRowKey().getValues(), + query.getSelectColumns().toArray((new String[0]))); + } else { + throw new ObTableException("unknown operation " + operation); + } + } + batchOps.setEntityType(entityType); + batchOps.setAtomicOperation(isAtomic); + batchOps.setReturnOneResult(returnOneResult); + return new BatchOperationResult(batchOps.executeWithResult()); + } + + private BatchOperationResult executeWithLSBatchOp() throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + ObTableClientLSBatchOpsImpl batchOps; + boolean hasSetRowkeyElement = false; + int checkAndInsUpCnt = 0; + + if (client instanceof ObTableClient) { + batchOps = new ObTableClientLSBatchOpsImpl(tableName, (ObTableClient) client); + for (Object operation : operations) { + if (operation instanceof CheckAndInsUp) { + checkAndInsUpCnt++; + CheckAndInsUp checkAndInsUp = (CheckAndInsUp) operation; + batchOps.addOperation(checkAndInsUp); + List rowKeyNames = checkAndInsUp.getInsUp().getRowKeyNames(); + if (!hasSetRowkeyElement && rowKeyNames != null) { + ((ObTableClient) client).addRowKeyElement(tableName, + rowKeyNames.toArray(new String[0])); + hasSetRowkeyElement = true; + } + } else if (operation instanceof Mutation) { + Mutation mutation = (Mutation) operation; + if (((ObTableClient) client).getRunningMode() == ObTableClient.RunningMode.HBASE) { + negateHbaseTimestamp(mutation); + } + batchOps.addOperation(mutation); + if (!hasSetRowkeyElement && mutation.getRowKeyNames() != null) { + List rowKeyNames = mutation.getRowKeyNames(); + ((ObTableClient) client).addRowKeyElement(tableName, + rowKeyNames.toArray(new String[0])); + hasSetRowkeyElement = true; + } + } else if (operation instanceof TableQuery) { + TableQuery query = (TableQuery) operation; + batchOps.addOperation(query); + } else { + throw new IllegalArgumentException( + "The operations in batch must be all checkAndInsUp or all non-checkAndInsUp"); + } + } + } else { + throw new IllegalArgumentException( + "execute batch using ObTable directly is not supported"); + } + + if (checkAndInsUpCnt > 0 && checkAndInsUpCnt != operations.size()) { + throw new IllegalArgumentException( + "Can not mix checkAndInsUP and other types operation in batch"); + } + + batchOps.setReturningAffectedEntity(withResult); + batchOps.setReturnOneResult(returnOneResult); + batchOps.setAtomicOperation(isAtomic); + batchOps.setEntityType(entityType); + return new BatchOperationResult(batchOps.executeWithResult()); + } + + private void negateHbaseTimestamp(Mutation mutation) { + Row rowKey = mutation.getRowKey(); + if (rowKey == null || rowKey.size() != 3) { + throw new IllegalArgumentException("hbase rowkey length must be 3"); + } else { + long ts = ((long) ((ObObj) mutation.getRowKeyValues().get(2)).getValue()); + ((ObObj) mutation.getRowKeyValues().get(2)).setValue(-ts); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/ColumnValue.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/ColumnValue.java new file mode 100644 index 00000000..bd751a32 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/ColumnValue.java @@ -0,0 +1,51 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.exception.ObTableException; + +public class ColumnValue { + private String columnName; + private Object value; + + /* + * default constructor, columnName should not be empty + */ + public ColumnValue(String columnName, Object value) { + if (null == columnName || columnName.isEmpty()) { + throw new ObTableException("column name is null"); + } + + this.columnName = columnName; + this.value = value; + } + + /* + * get the columnName + */ + public String getColumnName() { + return columnName; + } + + /* + * get the value + */ + public Object getValue() { + return value; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Delete.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Delete.java new file mode 100644 index 00000000..e0eb352a --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Delete.java @@ -0,0 +1,75 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.table.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Delete extends Mutation { + /* + * default constructor + */ + public Delete() { + super(); + } + + /* + * construct with ObTableClient and String + */ + public Delete(Table client, String tableName) { + super(client, tableName); + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return ObTableOperationType.DEL; + } + + /* + * execute + */ + public MutationResult execute() throws Exception { + if (null == getTableName() || getTableName().isEmpty()) { + throw new ObTableException("table name is null"); + } else if (null == getClient()) { + throw new ObTableException("client is null"); + } + + if (null == getQuery()) { + // simple Insert, without filter + return new MutationResult(((ObTableClient) getClient()).deleteWithResult( + getTableName(), getRowKey(), getKeyRanges())); + } else { + // QueryAndDelete + ObTableOperation operation = ObTableOperation.getInstance(ObTableOperationType.DEL, + new Object[] {}, null, null); + return new MutationResult(((ObTableClient) getClient()).mutationWithFilter(getQuery(), + getRowKey(), getKeyRanges(), operation, false)); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Increment.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Increment.java new file mode 100644 index 00000000..fc88b643 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Increment.java @@ -0,0 +1,146 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.table.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Increment extends Mutation { + boolean withResult; + + /* + * default constructor + */ + public Increment() { + super(); + columns = new ArrayList(); + values = new ArrayList(); + withResult = false; + } + + /* + * construct with ObTableClient and String + */ + public Increment(Table client, String tableName) { + super(client, tableName); + columns = new ArrayList(); + values = new ArrayList(); + withResult = false; + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return ObTableOperationType.INCREMENT; + } + + /* + * get the mutated columns' name + */ + public String[] getColumns() { + return columns.toArray(new String[0]); + } + + /* + * get the mutated columns' value + */ + public Object[] getValues() { + return values.toArray(); + } + + /* + * add mutated Row + */ + public Increment addMutateRow(Row rows) { + if (null == rows) { + throw new IllegalArgumentException("Invalid null rowKey set into Update"); + } else if (0 == rows.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set mutate row into Update + for (Map.Entry entry : rows.getMap().entrySet()) { + columns.add(entry.getKey()); + values.add(entry.getValue()); + } + + return this; + } + + /* + * add mutated ColumnValues + */ + public Increment addMutateColVal(ColumnValue... columnValues) { + if (null == columnValues) { + throw new IllegalArgumentException("Invalid null columnValues set into Update"); + } + + // set mutate ColumnValue into Update + for (ColumnValue columnValue : columnValues) { + if (columns.contains(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in Row Key"); + } + columns.add(columnValue.getColumnName()); + values.add(columnValue.getValue()); + } + + return this; + } + + /* + * Remove rowkey from mutateColval + */ + public Increment removeRowkeyFromMutateColval() { + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + return this; + } + + /* + * execute + */ + public MutationResult execute() throws Exception { + if (null == getTableName() || getTableName().isEmpty()) { + throw new ObTableException("table name is null"); + } else if (null == getClient()) { + throw new ObTableException("client is null"); + } + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + if (null == getQuery()) { + // simple increment, without filter + return new MutationResult(((ObTableClient) getClient()).incrementWithResult( + getTableName(), getRowKey(), getKeyRanges(), columns.toArray(new String[0]), + values.toArray(), withResult)); + } else { + // QueryAndIncrement + ObTableOperation operation = ObTableOperation.getInstance( + ObTableOperationType.INCREMENT, new Object[] {}, columns.toArray(new String[0]), + values.toArray()); + return new MutationResult(((ObTableClient) getClient()).mutationWithFilter(getQuery(), + getRowKey(), getKeyRanges(), operation, withResult)); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Insert.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Insert.java new file mode 100644 index 00000000..903539bb --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Insert.java @@ -0,0 +1,171 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.table.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Insert extends Mutation { + /* + * default constructor + */ + public Insert() { + super(); + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * construct with ObTableClient and String + */ + public Insert(Table client, String tableName) { + super(client, tableName); + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * set the Row Key of mutation with Row and keep scan range + */ + @Override + public Insert setRowKey(Row rowKey) { + return setRowKeyOnly(rowKey); + } + + /* + * set the Row Key of mutation with ColumnValues and keep scan range + */ + @Override + public Insert setRowKey(ColumnValue... rowKey) { + return setRowKeyOnly(rowKey); + } + + /* + * add filter into mutation (use QueryAndMutate) and scan range + */ + public Insert setFilter(ObTableFilter filter) throws Exception { + return setFilterOnly(filter); + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return ObTableOperationType.INSERT; + } + + /* + * add mutated Row + */ + public Insert addMutateRow(Row rows) { + if (null == rows) { + throw new IllegalArgumentException("Invalid null rowKey set into Insert"); + } else if (0 == rows.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set mutate row into Insert + for (Map.Entry entry : rows.getMap().entrySet()) { + columns.add(entry.getKey()); + values.add(entry.getValue()); + } + + return this; + } + + /* + * get the mutated columns' name + */ + public String[] getColumns() { + return columns.toArray(new String[0]); + } + + /* + * get the mutated columns' value + */ + public Object[] getValues() { + return values.toArray(); + } + + /* + * add mutated ColumnValues + */ + public Insert addMutateColVal(ColumnValue... columnValues) throws Exception { + if (null == columnValues) { + throw new IllegalArgumentException("Invalid null columnValues set into Insert"); + } + + // set mutate row into Insert + for (ColumnValue columnValue : columnValues) { + if (columns.contains(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in Row Key"); + } + columns.add(columnValue.getColumnName()); + values.add(columnValue.getValue()); + } + + return this; + } + + /* + * Remove rowkey from mutateColval + */ + public Insert removeRowkeyFromMutateColval() { + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + return this; + } + + /* + * execute + */ + public MutationResult execute() throws Exception { + if (null == getTableName() || getTableName().isEmpty()) { + throw new ObTableException("table name is null"); + } else if (null == getClient()) { + throw new ObTableException("client is null"); + } + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + if (null == getQuery()) { + // simple Insert, without filter + return new MutationResult(((ObTableClient) getClient()).insertWithResult( + getTableName(), getRowKey(), getKeyRanges(), columns.toArray(new String[0]), + values.toArray())); + } else { + if (checkMutationWithFilter()) { + // QueryAndInsert + ObTableOperation operation = ObTableOperation.getInstance( + ObTableOperationType.INSERT, getRowKeyValues().toArray(), columns.toArray(new String[0]), + values.toArray()); + return new MutationResult(((ObTableClient) getClient()).mutationWithFilter( + getQuery(), getRowKey(), getKeyRanges(), operation, true)); + } else { + throw new ObTableUnexpectedException("should set filter and scan range both"); + } + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/InsertOrUpdate.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/InsertOrUpdate.java new file mode 100644 index 00000000..f50415ed --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/InsertOrUpdate.java @@ -0,0 +1,148 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.table.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class InsertOrUpdate extends Mutation { + private boolean usePut; + + /* + * default constructor + */ + public InsertOrUpdate() { + super(); + usePut = false; + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * construct with ObTableClient and String + */ + public InsertOrUpdate(Table client, String tableName) { + super(client, tableName); + usePut = false; + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * set use put + */ + protected InsertOrUpdate usePut() { + usePut = true; + return this; + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return ObTableOperationType.INSERT_OR_UPDATE; + } + + /* + * get the mutated columns' name + */ + public String[] getColumns() { + return columns.toArray(new String[0]); + } + + /* + * get the mutated columns' value + */ + public Object[] getValues() { + return values.toArray(); + } + + /* + * add mutated Row + */ + public InsertOrUpdate addMutateRow(Row rows) { + if (null == rows) { + throw new IllegalArgumentException("Invalid null rowKey set into Insert"); + } else if (0 == rows.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set mutate row into Insert + for (Map.Entry entry : rows.getMap().entrySet()) { + columns.add(entry.getKey()); + values.add(entry.getValue()); + } + + return this; + } + + /* + * add mutated ColumnValues + */ + public InsertOrUpdate addMutateColVal(ColumnValue... columnValues) { + if (null == columnValues) { + throw new IllegalArgumentException("Invalid null columnValues set into Insert"); + } + + // set mutate row into Insert + for (ColumnValue columnValue : columnValues) { + if (columns.contains(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in Row Key"); + } + columns.add(columnValue.getColumnName()); + values.add(columnValue.getValue()); + } + + return this; + } + + /* + * Remove rowkey from mutateColval + */ + public InsertOrUpdate removeRowkeyFromMutateColval() { + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + return this; + } + + /* + * execute + */ + public MutationResult execute() throws Exception { + if (null == getTableName() || getTableName().isEmpty()) { + throw new ObTableException("table name is null"); + } else if (null == getClient()) { + throw new ObTableException("client is null"); + } + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + if (null == getQuery()) { + // simple InsertOrUpdate, without filter + return new MutationResult(((ObTableClient) getClient()).insertOrUpdateWithResult( + getTableName(), getRowKey(), getKeyRanges(), columns.toArray(new String[0]), + values.toArray(), usePut)); + } else { + throw new ObTableException("InsertOrUpdate with query(filter) is not supported yet"); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Mutation.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Mutation.java new file mode 100644 index 00000000..e099a6ab --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Mutation.java @@ -0,0 +1,493 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.filter.*; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjMeta; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObITableEntity; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntity; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; +import com.alipay.oceanbase.rpc.table.api.Table; +import com.alipay.oceanbase.rpc.table.api.TableQuery; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.row; + +public class Mutation { + private String tableName; + private Table client; + protected Row rowKey; + private TableQuery query; + private boolean hasSetRowKey = false; + protected List rowKeyNames = null; + protected List rowKeyValues = null; + protected List columns; + protected List values; + + /* + * default constructor + * recommend for batch operation + */ + public Mutation() { + tableName = null; + client = null; + rowKey = null; + query = null; + rowKeyNames = null; + rowKeyValues = null; + columns = null; + values = null; + } + + /* + * construct Mutation with client and tableName + */ + public Mutation(Table client, String tableName) { + if (null == client || null == tableName || tableName.isEmpty()) { + throw new IllegalArgumentException("Invalid input to create Mutation in table" + + tableName); + } + + this.client = client; + this.tableName = tableName; + this.rowKey = null; + this.query = null; + this.rowKeyNames = null; + this.rowKeyValues = null; + this.columns = null; + this.values = null; + } + + /* + * get client + */ + protected Table getClient() { + return client; + } + + /* + * get tableName + */ + protected String getTableName() { + return tableName; + } + + /* + * get query + */ + protected TableQuery getQuery() { + return query; + } + + /* + * get row key + */ + public Row getRowKey() { + return rowKey; + } + + /* + * get key ranges + */ + protected List getKeyRanges() { + if (null != query) { + return query.getObTableQuery().getKeyRanges(); + } + return null; + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return null; + } + + /* + * get rowkey names + */ + public List getRowKeyNames() { + return rowKeyNames; + } + + /* + * get rowkey values + */ + public List getRowKeyValues() { return rowKeyValues; } + + /* + * check mutation filter + */ + protected boolean checkMutationWithFilter() { + if (null == query) { + return false; + } + return query.getObTableQuery().isFilterNull(); + } + + /* + * set client + */ + @SuppressWarnings("unchecked") + public T setClient(ObTableClient client) { + if (null == client) { + throw new IllegalArgumentException("Invalid client to create Mutation"); + } + + this.client = client; + + return (T) this; + } + + /* + * set table + */ + @SuppressWarnings("unchecked") + public T setTable(String tableName) { + if (null == tableName || tableName.isEmpty()) { + throw new IllegalArgumentException("Invalid table name to create Mutation in table" + + tableName); + } + + this.tableName = tableName; + + return (T) this; + } + + /* + * set the Row Key of mutation with Row + */ + @SuppressWarnings("unchecked") + public T setRowKey(Row rowKey) { + if (hasSetRowKey) { + throw new IllegalArgumentException("Could not set row key (scan range) twice"); + } else if (null == rowKey) { + throw new IllegalArgumentException("Invalid null rowKey set into Mutation"); + } else if (0 == rowKey.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set rowKey + this.rowKey = rowKey; + + // set row key name into client and set rowKeys + this.rowKeyValues = new ArrayList<>(Arrays.asList(rowKey.getValues())); + this.rowKeyNames = new ArrayList<>(Arrays.asList(rowKey.getColumns())); + + // set row key in table + if (null != tableName) { + ((ObTableClient) client) + .addRowKeyElement(tableName, this.rowKeyNames.toArray(new String[0])); + } + + // renew scan range of QueryAndMutate + if (null != query) { + query.addScanRange(rowKeyValues.toArray(), rowKeyValues.toArray()); + } + + hasSetRowKey = true; + return (T) this; + } + + /* + * Set the Row Key of mutation with Row and keep scan range + */ + @SuppressWarnings("unchecked") + protected T setRowKeyOnly(Row rowKey) { + if (hasSetRowKey) { + throw new IllegalArgumentException("Could not set row key (scan range) twice"); + } else if (null == rowKey) { + throw new IllegalArgumentException("Invalid null rowKey set into Mutation"); + } else if (0 == rowKey.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set rowKey + this.rowKey = rowKey; + + // set row key name into client and set rowKeys + this.rowKeyValues = new ArrayList<>(Arrays.asList(rowKey.getValues())); + this.rowKeyNames = new ArrayList<>(Arrays.asList(rowKey.getColumns())); + + // set row key in table + if (null != tableName) { + ((ObTableClient) client) + .addRowKeyElement(tableName, this.rowKeyNames.toArray(new String[0])); + } + + hasSetRowKey = true; + return (T) this; + } + + /* + * set the Row Key of mutation with ColumnValues + */ + @SuppressWarnings("unchecked") + public T setRowKey(ColumnValue... rowKey) { + if (hasSetRowKey) { + throw new IllegalArgumentException("Could not set row key (scan range) twice"); + } else if (null == rowKey) { + throw new IllegalArgumentException("Invalid null rowKey set into Mutation"); + } + + // set rowKey + this.rowKey = new Row(rowKey); + + // set row key name into client and set rowKey + this.rowKeyValues = new ArrayList<>(Arrays.asList(this.rowKey.getValues())); + this.rowKeyNames = new ArrayList<>(Arrays.asList(this.rowKey.getColumns())); + + // set row key in table + if (null != tableName) { + ((ObTableClient) client) + .addRowKeyElement(tableName, this.rowKeyNames.toArray(new String[0])); + } + + // renew scan range of QueryAndMutate + if (null != query) { + query.addScanRange(rowKeyValues.toArray(), rowKeyValues.toArray()); + } + + hasSetRowKey = true; + return (T) this; + } + + /* + * set the Row Key of mutation with ColumnValues and keep scan range + */ + @SuppressWarnings("unchecked") + public T setRowKeyOnly(ColumnValue... rowKey) { + if (hasSetRowKey) { + throw new IllegalArgumentException("Could not set row key (scan range) twice"); + } else if (null == rowKey) { + throw new IllegalArgumentException("Invalid null rowKey set into Mutation"); + } + + // set rowKey + this.rowKey = new Row(rowKey); + + // set row key name into client and set rowKey + this.rowKeyValues = new ArrayList<>(Arrays.asList(this.rowKey.getValues())); + this.rowKeyNames = new ArrayList<>(Arrays.asList(this.rowKey.getColumns())); + + // set row key in table + if (null != tableName) { + ((ObTableClient) client) + .addRowKeyElement(tableName, this.rowKeyNames.toArray(new String[0])); + } + + hasSetRowKey = true; + return (T) this; + } + + /* + * add filter into mutation (use QueryAndMutate) + */ + @SuppressWarnings("unchecked") + public T setFilter(ObTableFilter filter) throws Exception { + if (null == filter) { + throw new IllegalArgumentException("Invalid null filter set into Mutation"); + } else if (null == client) { + // do nothing + } else { + if (null == query) { + query = client.query(tableName); + // set scan range if rowKey exist + if (null != rowKey) { + query.addScanRange(this.rowKeyValues.toArray(), this.rowKeyValues.toArray()); + } + } + // only filter string in query works + query.setFilter(filter); + } + return (T) this; + } + + /* + * add filter into mutation (use QueryAndMutate) and scan range + */ + @SuppressWarnings("unchecked") + public T setFilterOnly(ObTableFilter filter) throws Exception { + if (null == filter) { + throw new IllegalArgumentException("Invalid null filter set into Mutation"); + } else if (null == client) { + // do nothing + } else { + if (null == query) { + query = client.query(tableName); + } + // only filter string in query works + query.setFilter(filter); + } + return (T) this; + } + + /* + * used for scan range (not ODP) + */ + @SuppressWarnings("unchecked") + public T setScanRangeColumns(String... columnNames) throws Exception { + if (null == columnNames) { + throw new IllegalArgumentException("Invalid null column names set into Mutation"); + } + + if (null == query) { + query = client.query(tableName); + } + + query.setScanRangeColumns(columnNames); + + // set row key in table + if (null != tableName && null != client) { + if (!((ObTableClient) client).isOdpMode()) { + // TODO: adapt OCP + // OCP must conclude all rowkey now + ((ObTableClient) client).addRowKeyElement(tableName, columnNames); + } + } else { + throw new ObTableException("invalid table name: " + tableName + ", or invalid client: " + + client + " while setting scan range columns"); + } + + return (T) this; + } + + /* + * add scan range + */ + @SuppressWarnings("unchecked") + public T addScanRange(Object start, Object end) throws Exception { + if (null == start || null == end) { + throw new IllegalArgumentException("Invalid null range set into Mutation"); + } + + return addScanRange(new Object[] { start }, true, new Object[] { end }, true); + } + + /* + * add list of scan range + */ + @SuppressWarnings("unchecked") + public T addScanRange(Object[] start, Object[] end) throws Exception { + if (null == start || null == end) { + throw new IllegalArgumentException("Invalid null range set into Mutation"); + } + + return addScanRange(start, true, end, true); + } + + /* + * add scan range with boundary + */ + @SuppressWarnings("unchecked") + public T addScanRange(Object start, boolean startEquals, Object end, boolean endEquals) + throws Exception { + if (null == start || null == end) { + throw new IllegalArgumentException("Invalid null range set into Mutation"); + } + + return addScanRange(new Object[] { start }, startEquals, new Object[] { end }, endEquals); + } + + /* + * add list of scan range with boundary + */ + @SuppressWarnings("unchecked") + public T addScanRange(Object[] start, boolean startEquals, Object[] end, boolean endEquals) + throws Exception { + if (null == start || null == end) { + throw new IllegalArgumentException("Invalid null range set into Mutation"); + } + + if (null == query) { + query = client.query(tableName); + } + + query.addScanRange(start, startEquals, end, endEquals); + + return (T) this; + } + + static void removeRowkeyFromMutateColval(List columns, List values, + List rowKeyNames) { + if (null == columns || null == rowKeyNames || columns.size() != values.size()) { + return; + } + for (int i = values.size() - 1; i >= 0; --i) { + if (rowKeyNames.contains(columns.get(i))) { + columns.remove(i); + values.remove(i); + } + } + } + + public void addColVal(String propName, ObObj propValue) { + columns.add(propName); + values.add(propValue); + } + + public static Mutation getInstance(ObTableOperationType type, String[] rowKeyNames, + Object[] rowKeys, String[] columns, Object[] properties) { + Mutation mutation = null; + switch (type) { + case INSERT_OR_UPDATE: + mutation = new InsertOrUpdate(); + break; + case DEL: + mutation = new Delete(); + break; + default: + throw new ObTableException("not support operation type " + type); + } + + Row rowKeyRow = new Row(); + if (rowKeys != null) { + for (int i = 0; i < rowKeys.length; i++) { + Object rowkey = rowKeys[i]; + ObObj obj = ObObj.getInstance(rowkey); + rowKeyRow.add(rowKeyNames[i], obj); + } + } + mutation.setRowKey(rowKeyRow); + + if (columns != null) { + for (int i = 0; i < columns.length; i++) { + String name = columns[i]; + Object value = null; + if (properties != null) { + value = properties[i]; + } + ObObj c = ObObj.getInstance(value); + mutation.addColVal(name, c); + } + } + + return mutation; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/MutationFactory.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/MutationFactory.java new file mode 100644 index 00000000..5c4747b3 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/MutationFactory.java @@ -0,0 +1,73 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObClusterTableQuery; +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.table.ObTableClientQueryImpl; +import com.alipay.oceanbase.rpc.table.api.TableQuery; + +import javax.management.Query; + +public class MutationFactory { + public static ColumnValue colVal(String columnName, Object value) { + return new ColumnValue(columnName, value); + } + + public static Row row(ColumnValue... columnValue) { + return new Row(columnValue); + } + + public static TableQuery query() { + ObTableClientQueryImpl tableQuery = new ObTableClientQueryImpl(); + return new ObClusterTableQuery(tableQuery); + } + + public static Insert insert() { + return new Insert(); + } + + public static Delete delete() { + return new Delete(); + } + + public static Update update() { + return new Update(); + } + + public static InsertOrUpdate insertOrUpdate() { + return new InsertOrUpdate(); + } + + public static Put put() { + return new Put(); + } + + public static Replace replace() { + return new Replace(); + } + + public static Increment increment() { + return new Increment(); + } + + public static Append append() { + return new Append(); + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java new file mode 100644 index 00000000..580cde63 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Put.java @@ -0,0 +1,169 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.table.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Put extends Mutation { + /* + * default constructor + */ + public Put() { + super(); + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * construct with ObTableClient and String + */ + public Put(Table client, String tableName) { + super(client, tableName); + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * set the Row Key of mutation with Row and keep scan range + */ + @Override + public Put setRowKey(Row rowKey) { + return setRowKeyOnly(rowKey); + } + + /* + * set the Row Key of mutation with ColumnValues and keep scan range + */ + @Override + public Put setRowKey(ColumnValue... rowKey) { + return setRowKeyOnly(rowKey); + } + + /* + * add filter into mutation (use QueryAndMutate) and scan range + */ + public Put setFilter(ObTableFilter filter) throws Exception { + return setFilterOnly(filter); + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return ObTableOperationType.PUT; + } + + /* + * add mutated Row + */ + public Put addMutateRow(Row rows) { + if (null == rows) { + throw new IllegalArgumentException("Invalid null rowKey set into Put"); + } else if (0 == rows.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set mutate row into Put + for (Map.Entry entry : rows.getMap().entrySet()) { + columns.add(entry.getKey()); + values.add(entry.getValue()); + } + + return this; + } + + /* + * get the mutated columns' name + */ + public String[] getColumns() { + return columns.toArray(new String[0]); + } + + /* + * get the mutated columns' value + */ + public Object[] getValues() { + return values.toArray(); + } + + /* + * add mutated ColumnValues + */ + public Put addMutateColVal(ColumnValue... columnValues) throws Exception { + if (null == columnValues) { + throw new IllegalArgumentException("Invalid null columnValues set into Put"); + } + + // set mutate row into Put + for (ColumnValue columnValue : columnValues) { + if (columns.contains(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in Row Key"); + } + columns.add(columnValue.getColumnName()); + values.add(columnValue.getValue()); + } + + return this; + } + + /* + * Remove rowkey from mutateColval + */ + public Put removeRowkeyFromMutateColval() { + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + return this; + } + + /* + * execute + */ + public MutationResult execute() throws Exception { + if (null == getTableName()) { + throw new ObTableException("table name is null"); + } else if (null == getClient()) { + throw new ObTableException("client is null"); + } + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + if (null == getQuery()) { + // simple Put, without filter + return new MutationResult(((ObTableClient) getClient()).putWithResult(getTableName(), + getRowKey(), getKeyRanges(), columns.toArray(new String[0]), values.toArray())); + } else { + if (checkMutationWithFilter()) { + // QueryAndPut + ObTableOperation operation = ObTableOperation.getInstance(ObTableOperationType.PUT, + getRowKeyValues().toArray(), columns.toArray(new String[0]), values.toArray()); + return new MutationResult(((ObTableClient) getClient()).mutationWithFilter( + getQuery(), getRowKey(), getKeyRanges(), operation, true)); + } else { + throw new ObTableUnexpectedException("should set filter and scan range both"); + } + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Replace.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Replace.java new file mode 100644 index 00000000..57aac160 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Replace.java @@ -0,0 +1,136 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.table.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Replace extends Mutation { + /* + * default constructor + */ + public Replace() { + super(); + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * construct with ObTableClient and String + */ + public Replace(Table client, String tableName) { + super(client, tableName); + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * get the mutated columns' name + */ + public String[] getColumns() { + return columns.toArray(new String[0]); + } + + /* + * get the mutated columns' value + */ + public Object[] getValues() { + return values.toArray(); + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return ObTableOperationType.REPLACE; + } + + /* + * add mutated Row + */ + public Replace addMutateRow(Row rows) { + if (null == rows) { + throw new IllegalArgumentException("Invalid null rowKey set into Insert"); + } else if (0 == rows.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set mutate row into Insert + for (Map.Entry entry : rows.getMap().entrySet()) { + columns.add(entry.getKey()); + values.add(entry.getValue()); + } + + return this; + } + + /* + * add mutated ColumnValues + */ + public Replace addMutateColVal(ColumnValue... columnValues) { + if (null == columnValues) { + throw new IllegalArgumentException("Invalid null columnValues set into Insert"); + } + + // set mutate row into Insert + for (ColumnValue columnValue : columnValues) { + if (columns.contains(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in Row Key"); + } + columns.add(columnValue.getColumnName()); + values.add(columnValue.getValue()); + } + + return this; + } + + /* + * Remove rowkey from mutateColval + */ + public Replace removeRowkeyFromMutateColval() { + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + return this; + } + + /* + * execute + */ + public MutationResult execute() throws Exception { + if (null == getTableName() || getTableName().isEmpty()) { + throw new ObTableException("table name is null"); + } else if (null == getClient()) { + throw new ObTableException("client is null"); + } + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + if (null == getQuery()) { + // simple replace, without filter + return new MutationResult(((ObTableClient) getClient()).replaceWithResult( + getTableName(), getRowKey(), getKeyRanges(), columns.toArray(new String[0]), + values.toArray())); + } else { + throw new ObTableException("Replace with query(filter) is not supported yet"); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Row.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Row.java new file mode 100644 index 00000000..a3bae5bd --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Row.java @@ -0,0 +1,141 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.exception.ObTableException; + +import java.util.*; + +public class Row { + private Map values; + + /* + * default constructor + */ + public Row() { + values = new LinkedHashMap(); + } + + /* + * construct with map + */ + public Row(Map values) { + if (null == values) { + throw new ObTableException("input of Row is null"); + } + this.values = values; + } + + /* + * construct with columnName and value + */ + public Row(String columnName, Object value) { + if (null == columnName || columnName.isEmpty()) { + throw new ObTableException("column name is null"); + } + + values = new LinkedHashMap(); + values.put(columnName, value); + } + + /* + * construct with ColumnValue + */ + public Row(ColumnValue... columnValues) { + values = new LinkedHashMap(); + + for (ColumnValue columnValue : columnValues) { + if (values.containsKey(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in row"); + } + + values.put(columnValue.getColumnName(), columnValue.getValue()); + } + } + + /* + * add column with String and Object + */ + public Row add(String columnName, Object value) { + if (null == columnName || columnName.isEmpty()) { + throw new ObTableException("column name is null"); + } + + values.put(columnName, value); + + return this; + } + + /* + * add column with ColumnValue + */ + public Row add(ColumnValue... columnValues) { + for (ColumnValue columnValue : columnValues) { + if (values.containsKey(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in row"); + } + + values.put(columnValue.getColumnName(), columnValue.getValue()); + } + + return this; + } + + /* + * get columns' name + */ + public String[] getColumns() { + List keys = new ArrayList(); + for (Map.Entry entry : values.entrySet()) { + keys.add((entry.getKey())); + } + return keys.toArray(new String[0]); + } + + /* + * get values + */ + public Object[] getValues() { + List keys = new ArrayList(); + for (Map.Entry entry : values.entrySet()) { + keys.add((entry.getValue())); + } + return keys.toArray(); + } + + /* + * get the column-value map + */ + public Map getMap() { + return values; + } + + /* + * get value from key + */ + public Object get(String key) { + return values.get(key); + } + + /* + * size of row + */ + public long size() { + return values.size(); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/Update.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/Update.java new file mode 100644 index 00000000..7d392668 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/Update.java @@ -0,0 +1,141 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.table.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Update extends Mutation { + /* + * default constructor + */ + public Update() { + super(); + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * construct with ObTableClient and String + */ + public Update(Table client, String tableName) { + super(client, tableName); + columns = new ArrayList(); + values = new ArrayList(); + } + + /* + * get operation type + */ + public ObTableOperationType getOperationType() { + return ObTableOperationType.UPDATE; + } + + /* + * get the mutated columns' name + */ + public String[] getColumns() { + return columns.toArray(new String[0]); + } + + /* + * get the mutated columns' value + */ + public Object[] getValues() { + return values.toArray(); + } + + /* + * add mutated Row + */ + public Update addMutateRow(Row rows) { + if (null == rows) { + throw new IllegalArgumentException("Invalid null rowKey set into Update"); + } else if (0 == rows.getMap().size()) { + throw new IllegalArgumentException("input row key should not be empty"); + } + + // set mutate row into Update + for (Map.Entry entry : rows.getMap().entrySet()) { + columns.add(entry.getKey()); + values.add(entry.getValue()); + } + + return this; + } + + /* + * add mutated ColumnValues + */ + public Update addMutateColVal(ColumnValue... columnValues) { + if (null == columnValues) { + throw new IllegalArgumentException("Invalid null columnValues set into Update"); + } + + // set mutate ColumnValue into Update + for (ColumnValue columnValue : columnValues) { + if (columns.contains(columnValue.getColumnName())) { + throw new ObTableException("Duplicate column in Row Key"); + } + columns.add(columnValue.getColumnName()); + values.add(columnValue.getValue()); + } + + return this; + } + + /* + * Remove rowkey from mutateColval + */ + public Update removeRowkeyFromMutateColval() { + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + return this; + } + + /* + * execute + */ + public MutationResult execute() throws Exception { + if (null == getTableName() || getTableName().isEmpty()) { + throw new ObTableException("table name is null"); + } else if (null == getClient()) { + throw new ObTableException("client is null"); + } + removeRowkeyFromMutateColval(this.columns, this.values, this.rowKeyNames); + if (null == getQuery()) { + // simple update, without filter + return new MutationResult(((ObTableClient) getClient()).updateWithResult( + getTableName(), getRowKey(), getKeyRanges(), columns.toArray(new String[0]), + values.toArray())); + } else { + // QueryAndUpdate + ObTableOperation operation = ObTableOperation.getInstance(ObTableOperationType.UPDATE, + new Object[] {}, columns.toArray(new String[0]), values.toArray()); + return new MutationResult(((ObTableClient) getClient()).mutationWithFilter(getQuery(), + getRowKey(), getKeyRanges(), operation, false)); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/result/BatchOperationResult.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/result/BatchOperationResult.java new file mode 100644 index 00000000..2073aa62 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/result/BatchOperationResult.java @@ -0,0 +1,155 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation.result; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; + +import java.util.ArrayList; +import java.util.List; + +public class BatchOperationResult { + + private List results; + + boolean hasError = false; + + /* + * construct with List of Object + */ + public BatchOperationResult(List results) { + this.results = results; + } + + public List getResults() { + return results; + } + + public List getErrorCodeList() { + List errorCodeList = new ArrayList(); + for (Object item : results) { + int errorCode = ResultCodes.OB_SUCCESS.errorCode; + if (item instanceof ObTableException) { + errorCode = ((ObTableException) item).getErrorCode(); + hasError = true; + } + errorCodeList.add(errorCode); + } + return errorCodeList; + } + + public boolean hasError() { + if (!hasError) { + for (Object item : results) { + if (item instanceof ObTableException) { + hasError = true; + break; + } + } + } + return hasError; + } + + public ObTableException getFirstException() { + ObTableException exception = null; + for (Object item : results) { + if (item instanceof ObTableException) { + exception = (ObTableException) item; + hasError = true; + break; + } + } + return exception; + } + + /* + * get result + */ + public OperationResult get(int pos) { + if (pos >= results.size()) { + throw new IllegalArgumentException("Invalid pos: " + pos + + ", while size of results is: " + results.size()); + } + return (MutationResult) results.get(pos); + } + + /* + * get size + */ + public int size() { + return results.size(); + } + + /* + * get wrong count in result + */ + public long getWrongCount() { + long wrongCount = 0; + for (Object item : results) { + if (item instanceof ObTableException) { + ++wrongCount; + hasError = true; + } + } + return wrongCount; + } + + /* + * get correct count in result + */ + public long getCorrectCount() { + long correctCount = 0; + for (Object item : results) { + if (!(item instanceof ObTableException)) { + ++correctCount; + } + } + return correctCount; + } + + /* + * get wrong idx in result + */ + public int[] getWrongIdx() { + List wrongIdx = new ArrayList(); + Integer i = 0; + for (Object item : results) { + if (item instanceof ObTableException) { + wrongIdx.add(i); + hasError = true; + } + ++i; + } + return wrongIdx.stream().mapToInt(Integer::intValue).toArray(); + } + + /* + * get wrong count in result + */ + public int[] getCorrectIdx() { + List correctIdx = new ArrayList(); + Integer i = 0; + for (Object item : results) { + if (!(item instanceof ObTableException)) { + correctIdx.add(i); + } + ++i; + } + return correctIdx.stream().mapToInt(Integer::intValue).toArray(); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/result/MutationResult.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/result/MutationResult.java new file mode 100644 index 00000000..618113d3 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/result/MutationResult.java @@ -0,0 +1,90 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation.result; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.Row; +import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableSingleOpResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateResult; + +import java.util.Map; + +public class MutationResult extends OperationResult { + /* + * construct with ObPayload + */ + public MutationResult(ObPayload result) { + super(result); + } + + /* + * get operation type of result + */ + public ObTableOperationType getOperationType() { + if (result instanceof ObTableSingleOpResult) { + return ((ObTableSingleOpResult) result).getOperationType(); + } + return ((ObTableOperationResult) result).getOperationType(); + } + + /* + * get the affected rows of mutation + */ + public long getAffectedRows() { + long affectedRows = 0; + switch (result.getPcode()) { + case Pcodes.OB_TABLE_API_EXECUTE: + affectedRows = ((ObTableOperationResult) result).getAffectedRows(); + break; + case Pcodes.OB_TABLE_API_QUERY_AND_MUTATE: + affectedRows = ((ObTableQueryAndMutateResult) result).getAffectedRows(); + break; + case Pcodes.OB_TABLE_API_LS_EXECUTE: + affectedRows = ((ObTableSingleOpResult) result).getAffectedRows(); + break; + default: + throw new ObTableException("unknown result type: " + result.getPcode()); + } + return affectedRows; + } + + /* + * get the result rows of operation + */ + public Row getOperationRow() { + Map rowsMap; + switch (result.getPcode()) { + case Pcodes.OB_TABLE_API_EXECUTE: + rowsMap = ((ObTableOperationResult) result).getEntity().getSimpleProperties(); + break; + case Pcodes.OB_TABLE_API_QUERY_AND_MUTATE: + throw new ObTableException("could not get query and mutate result now" + + result.getPcode()); + case Pcodes.OB_TABLE_API_LS_EXECUTE: + rowsMap = ((ObTableSingleOpResult) result).getEntity().getSimpleProperties(); + break; + default: + throw new ObTableException("unknown result type: " + result.getPcode()); + } + return new Row(rowsMap); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/mutation/result/OperationResult.java b/src/main/java/com/alipay/oceanbase/rpc/mutation/result/OperationResult.java new file mode 100644 index 00000000..162df27c --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/mutation/result/OperationResult.java @@ -0,0 +1,58 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.mutation.result; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.Row; +import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateResult; + +import java.util.Map; + +public class OperationResult { + + protected final ObPayload result; + + /* + * construct with ObPayload + */ + public OperationResult(ObPayload result) { + if (null == result) { + throw new IllegalArgumentException("Invalid null result"); + } + + this.result = result; + } + + /* + * get the affected rows of operation + */ + public long getAffectedRows() { + return -1; + } + + /* + * get the result rows of operation + */ + public Row getOperationRow() { + return null; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/property/AbstractPropertyAware.java b/src/main/java/com/alipay/oceanbase/rpc/property/AbstractPropertyAware.java index 9f4fe02c..5ebe84cf 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/property/AbstractPropertyAware.java +++ b/src/main/java/com/alipay/oceanbase/rpc/property/AbstractPropertyAware.java @@ -24,14 +24,14 @@ public abstract class AbstractPropertyAware { protected Properties properties = new Properties(); - /** + /* * Parse to int. */ public int parseToInt(String key) throws NumberFormatException { return Integer.parseInt(System.getProperty(OB_TABLE_CLIENT_PREFIX + key, getProperty(key))); } - /** + /* * Parse to int. */ public int parseToInt(String key, int defaultV) { @@ -42,14 +42,14 @@ public int parseToInt(String key, int defaultV) { } } - /** + /* * Parse to long. */ public long parseToLong(String key) throws NumberFormatException { return Long.parseLong(System.getProperty(OB_TABLE_CLIENT_PREFIX + key, getProperty(key))); } - /** + /* * Parse to long. */ public long parseToLong(String key, long defaultV) { @@ -60,28 +60,41 @@ public long parseToLong(String key, long defaultV) { } } - /** + public boolean parseToBoolean(String key) { + return Boolean.parseBoolean(System.getProperty(OB_TABLE_CLIENT_PREFIX + key, + getProperty(key))); + } + + public boolean parseToBoolean(String key, boolean defaultV) { + try { + return parseToBoolean(key); + } catch (Exception e) { + return defaultV; + } + } + + /* * Get property. */ public String getProperty(String key) { return properties.getProperty(key); } - /** + /* * Add property. */ public void addProperty(String key, String value) { this.properties.put(key, value); } - /** + /* * Set properties. */ public void setProperties(Properties properties) { this.properties = properties; } - /** + /* * Get properties. */ public Properties getProperties() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/property/Property.java b/src/main/java/com/alipay/oceanbase/rpc/property/Property.java index 19509b46..841cc593 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/property/Property.java +++ b/src/main/java/com/alipay/oceanbase/rpc/property/Property.java @@ -20,34 +20,27 @@ import com.alipay.remoting.config.Configs; /** - * all default time is millisecond + * all default time is millisecond, should add property into initProperties() * */ public enum Property { + /* + * property in both [`ObTableClient`] & [`ObTable`] + */ + RPC_CONNECT_TIMEOUT("rpc.connect.timeout", 1000, "建立RPC连接的超时时间"), - //remote call related - RPC_CONNECT_TIMEOUT("rpc.connect.timeout", 500, "建立RPC连接的超时时间"), - - RPC_CONNECT_TRY_TIMES("rpc.connect.try.times", 3, "建立RPC连接的尝试次数"), - - RPC_EXECUTE_TIMEOUT("rpc.execute.timeout", 3000, "执行RPC请求的socket超时时间"), - - RPC_LOGIN_TIMEOUT("rpc.login.timeout", 1000, "请求RPC登录的超时时间"), - - RPC_LOGIN_TRY_TIMES("rpc.login.try.times", 3, "请求RPC登录的尝试次数"), - - RPC_OPERATION_TIMEOUT("rpc.operation.timeout", 10000L, "OB内部执行RPC请求的超时时间"), - + /* + * property in [`ObTableClient`] + */ + // [ObTableClient][METADATA] METADATA_REFRESH_INTERVAL("metadata.refresh.interval", 60000L, "刷新METADATA的时间间隔"), - /** - * typo of METADATA_REFRESH_INTERVAL - */ @Deprecated METADATA_REFRESH_INTERNAL("metadata.refresh.internal", 60000L, "刷新METADATA的时间间隔"), METADATA_REFRESH_LOCK_TIMEOUT("metadata.refresh.lock.timeout", 8000L, "刷新METADATA的锁超时时间"), + // [ObTableClient][RS_LIST] RS_LIST_ACQUIRE_CONNECT_TIMEOUT("rs.list.acquire.connect.timeout", 200, "获取RS列表的建连的超时时间"), RS_LIST_ACQUIRE_READ_TIMEOUT("rs.list.acquire.read.timeout", 1000, "获取RS列表的读取的超时时间"), @@ -56,12 +49,10 @@ public enum Property { RS_LIST_ACQUIRE_RETRY_INTERVAL("rs.list.acquire.retry.interval", 100L, "每次尝试获取RS列表的时间间隔"), - /** - * typo of RS_LIST_ACQUIRE_RETRY_INTERVAL - */ @Deprecated RS_LIST_ACQUIRE_RETRY_INTERNAL("rs.list.acquire.retry.internal", 100L, "每次尝试获取RS列表的时间间隔"), + // [ObTableClient][TABLE_ENTRY] TABLE_ENTRY_ACQUIRE_CONNECT_TIMEOUT("table.entry.acquire.connect.timeout", 500L, "刷新TABLE地址的建连超时时间"), @@ -70,18 +61,12 @@ public enum Property { TABLE_ENTRY_REFRESH_INTERVAL_BASE("table.entry.refresh.interval.base", 100L, "刷新TABLE地址的基础时间间隔"), - /** - * typo of TABLE_ENTRY_REFRESH_INTERVAL_BASE - */ @Deprecated TABLE_ENTRY_REFRESH_INTERNAL_BASE("table.entry.refresh.internal.base", 100L, "刷新TABLE地址的基础时间间隔"), TABLE_ENTRY_REFRESH_INTERVAL_CEILING("table.entry.refresh.interval.ceiling", 1600L, "刷新TABLE地址的最大时间间隔"), - /** - * typo of TABLE_ENTRY_REFRESH_INTERVAL_CEILING - */ @Deprecated TABLE_ENTRY_REFRESH_INTERNAL_CEILING("table.entry.refresh.internal.ceiling", 1600L, "刷新TABLE地址的最大时间间隔"), @@ -91,41 +76,70 @@ public enum Property { TABLE_ENTRY_REFRESH_LOCK_TIMEOUT("table.entry.refresh.lock.timeout", 4000L, "刷新TABLE地址的锁超时时间"), - TABLE_ENTRY_REFRESH_TYE_TIMES("table.entry.refresh.try.times", 3, "刷新TABLE地址的尝试次数"), + TABLE_ENTRY_REFRESH_TRY_TIMES("table.entry.refresh.try.times", 3, "刷新TABLE地址的尝试次数"), TABLE_ENTRY_REFRESH_CONTINUOUS_FAILURE_CEILING( "table.entry.refresh.continuous.failure.ceiling", 10, "连续刷新TABLE地址的失败上限，会刷新METADATA"), + // [ObTableClient][SERVER_ADDRESS] SERVER_ADDRESS_PRIORITY_TIMEOUT("server.address.priority.timeout", 1800000L, "SERVER地址优先级的失效时间"), - SERVER_CONNECTION_POOL_SIZE("server.connection.pool.size", 1, "单个SERVER的连接数"), - SERVER_ADDRESS_CACHING_TIMEOUT("server.address.caching.timeout", 3600000L, "SERVER地址缓存的失效时间"), - //runtime execute related RUNTIME_CONTINUOUS_FAILURE_CEILING("runtime.continuous.failure.ceiling", 100, "连续运行失败上限，会刷新TABLE的地址"), + // [ObTableClient][RUNTIME] + // property in both obTableClient / direct load client RUNTIME_RETRY_TIMES("runtime.retry.times", 1, "运行过程中遇到可重试错误时的重试次数"), + // property in both obTableClient / direct load client RUNTIME_RETRY_INTERVAL("runtime.retry.interval", 1, "运行出错时重试的时间间隔"), RUNTIME_MAX_WAIT("runtime.max.wait", 3000L, "单次执行超时时间会在超时时间内重试"), RUNTIME_BATCH_MAX_WAIT("runtime.batch.max.wait", 3000L, "批量执行请求的超时时间"), - RUNTIME_BATCH_EXECUTOR("runtime.batch.executor", null, "批量请求时并发执行的线程池"), + // [ObTableClient][LOG] + SLOW_QUERY_MONITOR_THRESHOLD("slow.query.monitor.threshold", 10L, "记录到 MONITOR 日志中的慢操作的运行时间阈值"), + + /* + * property in [`ObTable`] + */ + // [ObTable][RPC] + RPC_CONNECT_TRY_TIMES("rpc.connect.try.times", 3, "建立RPC连接的尝试次数"), + + RPC_EXECUTE_TIMEOUT("rpc.execute.timeout", 3000, "执行RPC请求的socket超时时间"), + + RPC_LOGIN_TIMEOUT("rpc.login.timeout", 1000, "请求RPC登录的超时时间"), + + RPC_LOGIN_TRY_TIMES("rpc.login.try.times", 3, "请求RPC登录的尝试次数"), + + RPC_OPERATION_TIMEOUT("rpc.operation.timeout", 10000L, "OB内部执行RPC请求的超时时间"), + + // [ObTable][CONNECTION_POOL] + SERVER_CONNECTION_POOL_SIZE("server.connection.pool.size", 1, "单个SERVER的连接数"), + // [ObTable][NETTY] // overwrite the global default netty watermark for ob table: [512K, 1M] NETTY_BUFFER_LOW_WATERMARK(Configs.NETTY_BUFFER_LOW_WATERMARK, 512 * 1024, "netty写缓存的低水位"), NETTY_BUFFER_HIGH_WATERMARK(Configs.NETTY_BUFFER_HIGH_WATERMARK, 1024 * 1024, "netty写缓存的高水位"), + // Theoretically with normal 10Gbps network card, it costs 0.5 ms to flush 512K, // that is, flush the buffer from high_watermark(1M) to low_watermark(512k) by default. // So before throw Exception when buffer is full, sleep 1ms by default for the scenario // when a big package is blocking the buffer but the server is OK. - NETTY_BLOCKING_WAIT_INTERVAL("bolt.netty.blocking.wait.interval", 1, "netty写缓存满后等待时间"); + NETTY_BLOCKING_WAIT_INTERVAL("bolt.netty.blocking.wait.interval", 1, "netty写缓存满后等待时间"), + + // [ObTable][OTHERS] + SERVER_ENABLE_REROUTING("server.enable.rerouting", "false", "开启server端的重定向回复功能"), + + /* + * other config + */ + RUNTIME_BATCH_EXECUTOR("runtime.batch.executor", null, "批量请求时并发执行的线程池"); private final String key; private final Object defaultV; @@ -137,42 +151,42 @@ public enum Property { this.desc = desc; } - /** + /* * Get key. */ public String getKey() { return key; } - /** + /* * Get default int. */ public int getDefaultInt() { return (Integer) defaultV; } - /** + /* * Get default long. */ public long getDefaultLong() { return (Long) defaultV; } - /** + /* * Get default object. */ public Object getDefaultObject() { return defaultV; } - /** + /* * Get default boolean. */ public boolean getDefaultBoolean() { return (Boolean) defaultV; } - /** + /* * Get desc. */ public String getDesc() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObCompressType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObCompressType.java index 16735b0e..68518e5c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObCompressType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObCompressType.java @@ -28,14 +28,14 @@ public enum ObCompressType { this.code = code; } - /** + /* * Get code. */ public int getCode() { return code; } - /** + /* * Value of. */ public static ObCompressType valueOf(int code) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcCostTime.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcCostTime.java index 3354094a..79fcd6c5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcCostTime.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcCostTime.java @@ -33,13 +33,13 @@ public class ObRpcCostTime { private long packetId; private long requestArrivalTime; - /** + /* * Ob rpc cost time. */ public ObRpcCostTime() { } - /** + /* * Encode. */ public byte[] encode() { @@ -64,7 +64,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ public Object decode(ByteBuf buf) { @@ -80,98 +80,98 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get arrival push diff. */ public int getArrivalPushDiff() { return arrivalPushDiff; } - /** + /* * Set arrival push diff. */ public void setArrivalPushDiff(int arrivalPushDiff) { this.arrivalPushDiff = arrivalPushDiff; } - /** + /* * Get push pop diff. */ public int getPushPopDiff() { return pushPopDiff; } - /** + /* * Set push pop diff. */ public void setPushPopDiff(int pushPopDiff) { this.pushPopDiff = pushPopDiff; } - /** + /* * Get pop process start diff. */ public int getPopProcessStartDiff() { return popProcessStartDiff; } - /** + /* * Set pop process start diff. */ public void setPopProcessStartDiff(int popProcessStartDiff) { this.popProcessStartDiff = popProcessStartDiff; } - /** + /* * Get process start end diff. */ public int getProcessStartEndDiff() { return processStartEndDiff; } - /** + /* * Set process start end diff. */ public void setProcessStartEndDiff(int processStartEndDiff) { this.processStartEndDiff = processStartEndDiff; } - /** + /* * Get process end response diff. */ public int getProcessEndResponseDiff() { return processEndResponseDiff; } - /** + /* * Set process end response diff. */ public void setProcessEndResponseDiff(int processEndResponseDiff) { this.processEndResponseDiff = processEndResponseDiff; } - /** + /* * Get packet id. */ public long getPacketId() { return packetId; } - /** + /* * Set packet id. */ public void setPacketId(long packetId) { this.packetId = packetId; } - /** + /* * Get request arrival time. */ public long getRequestArrivalTime() { return requestArrivalTime; } - /** + /* * Set request arrival time. */ public void setRequestArrivalTime(long requestArrivalTime) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcPacket.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcPacket.java index b83e19f3..e95bb964 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcPacket.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcPacket.java @@ -22,39 +22,39 @@ public class ObRpcPacket { private ObRpcPacketHeader rpcPacketHeader; private byte[] payloadContent; - /** + /* * Get rpc packet header. */ public ObRpcPacketHeader getRpcPacketHeader() { return rpcPacketHeader; } - /** + /* * Set rpc packet header. */ public void setRpcPacketHeader(ObRpcPacketHeader rpcPacketHeader) { this.rpcPacketHeader = rpcPacketHeader; } - /** + /* * Get payload content. */ public byte[] getPayloadContent() { return payloadContent; } - /** + /* * Set payload content. */ public void setPayloadContent(byte[] payloadContent) { this.payloadContent = payloadContent; } - /** + /* * useless, only for reference */ @Deprecated - /** + /* * Encode. */ public byte[] encode() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcPacketHeader.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcPacketHeader.java index 180c38bb..5057a89e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcPacketHeader.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/packet/ObRpcPacketHeader.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.rpc.protocol.packet; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; @@ -53,7 +54,19 @@ public class ObRpcPacketHeader { + 8 // 8 is clusterId + 4 // obCompressType + 4; // originalLen - + private static final int ENCODE_SIZE_V4 = HEADER_SIZE + + ObRpcCostTime.ENCODED_SIZE // + + 8 // 8 is dst clusterId + + 4 // obCompressType + + 4 // originalLen + + 8 // src clusterId + + 8 // unis version + + 4 // request level + + 8 // seq no + + 4 // group id + + 8 // trace id2 + + 8 // trace id3 + + 8; //clusterNameHash public static final int RESP_FLAG = 1 << 15; public static final int STREAM_FLAG = 1 << 14; @@ -64,9 +77,10 @@ public class ObRpcPacketHeader { public static final int CONTEXT_FLAG = 1 << 11; public static final int UNNEED_RESPONSE_FLAG = 1 << 10; public static final int REQUIRE_REROUTING_FLAG = 1 << 9; + public static final int IS_KV_REQUEST_FALG = 1 << 5; private int pcode; - private short hlen = (short) ENCODE_SIZE; + private short hlen = 0; private short priority = 5; private short flag = 0; @@ -80,13 +94,21 @@ public class ObRpcPacketHeader { .getDefaultLong() * 1000; // OB server timeout (us) private long timestamp = System.currentTimeMillis() * 1000; // us private ObRpcCostTime obRpcCostTime = new ObRpcCostTime(); - private long clusterId = -1; // FIXME + private long dstClusterId = -1; // FIXME private ObCompressType obCompressType = INVALID_COMPRESSOR; private int originalLen = 0; + private long srcClusterId = -1; + private long unisVersion = 0; + private int requestLevel = 0; + private long seqNo = 0; + private int groupId = 0; + private long traceId2; + private long traceId3; + private long clusterNameHash; - /** + /* * Ob rpc packet header. */ public ObRpcPacketHeader() { @@ -100,14 +122,23 @@ public ObRpcPacketHeader() { #define OB_LOG_LEVEL_TRACE 4 #define OB_LOG_LEVEL_DEBUG 5 */ - flag = 0x7; // let ObServer determine the ob log level. + flag = 0x7 | IS_KV_REQUEST_FALG; // 0x7 means let ObServer determine the ob log level. } - /** + /* * Encode. */ public byte[] encode() { - byte[] bytes = new byte[ENCODE_SIZE]; + byte[] bytes = null; + if (hlen != 0) { + bytes = new byte[ENCODE_SIZE]; + } else if (ObGlobal.obVsnMajor() >= 4) { + bytes = new byte[ENCODE_SIZE_V4]; + hlen = (short) ENCODE_SIZE_V4; + } else { + bytes = new byte[ENCODE_SIZE]; + hlen = (short) ENCODE_SIZE; + } int idx = 0; System.arraycopy(Serialization.encodeI32(pcode), 0, bytes, idx, 4); @@ -136,15 +167,34 @@ public byte[] encode() { idx += 8; System.arraycopy(obRpcCostTime.encode(), 0, bytes, idx, ObRpcCostTime.ENCODED_SIZE); idx += ObRpcCostTime.ENCODED_SIZE; - System.arraycopy(Serialization.encodeI64(clusterId), 0, bytes, idx, 8); + System.arraycopy(Serialization.encodeI64(dstClusterId), 0, bytes, idx, 8); idx += 8; System.arraycopy(Serialization.encodeI32(obCompressType.getCode()), 0, bytes, idx, 4); idx += 4; System.arraycopy(Serialization.encodeI32(originalLen), 0, bytes, idx, 4); + if (ObGlobal.obVsnMajor() >= 4 && hlen >= ENCODE_SIZE_V4) { + idx += 4; + System.arraycopy(Serialization.encodeI64(srcClusterId), 0, bytes, idx, 8); + idx += 8; + System.arraycopy(Serialization.encodeI64(unisVersion), 0, bytes, idx, 8); + idx += 8; + System.arraycopy(Serialization.encodeI32(requestLevel), 0, bytes, idx, 4); + idx += 4; + System.arraycopy(Serialization.encodeI64(seqNo), 0, bytes, idx, 8); + idx += 8; + System.arraycopy(Serialization.encodeI32(groupId), 0, bytes, idx, 4); + idx += 4; + System.arraycopy(Serialization.encodeI64(traceId2), 0, bytes, idx, 8); + idx += 8; + System.arraycopy(Serialization.encodeI64(traceId3), 0, bytes, idx, 8); + idx += 8; + System.arraycopy(Serialization.encodeI64(clusterNameHash), 0, bytes, idx, 8); + } + return bytes; } - /** + /* * Decode. */ public Object decode(ByteBuf buf) { @@ -161,15 +211,29 @@ public Object decode(ByteBuf buf) { this.timeout = Serialization.decodeI64(buf); this.timestamp = Serialization.decodeI64(buf); - if (hlen >= ENCODE_SIZE) { + if (hlen >= ENCODE_SIZE_V4) { + obRpcCostTime.decode(buf); + this.dstClusterId = Serialization.decodeI64(buf); + this.obCompressType = ObCompressType.valueOf(Serialization.decodeI32(buf)); + this.originalLen = Serialization.decodeI32(buf); + this.srcClusterId = Serialization.decodeI64(buf); + this.unisVersion = Serialization.decodeI64(buf); + this.requestLevel = Serialization.decodeI32(buf); + this.seqNo = Serialization.decodeI64(buf); + this.groupId = Serialization.decodeI32(buf); + this.traceId2 = Serialization.decodeI64(buf); + this.traceId3 = Serialization.decodeI64(buf); + this.clusterNameHash = Serialization.decodeI64(buf); + ignoreUnresolvedBytes(buf, hlen, ENCODE_SIZE_V4); + } else if (hlen >= ENCODE_SIZE) { obRpcCostTime.decode(buf); - this.clusterId = Serialization.decodeI64(buf); + this.dstClusterId = Serialization.decodeI64(buf); this.obCompressType = ObCompressType.valueOf(Serialization.decodeI32(buf)); this.originalLen = Serialization.decodeI32(buf); ignoreUnresolvedBytes(buf, hlen, ENCODE_SIZE); } else if (hlen >= ENCODE_SIZE_WITH_COST_TIME_AND_CLUSTER_ID) { obRpcCostTime.decode(buf); - this.clusterId = Serialization.decodeI64(buf); + this.dstClusterId = Serialization.decodeI64(buf); ignoreUnresolvedBytes(buf, hlen, ENCODE_SIZE_WITH_COST_TIME_AND_CLUSTER_ID); } else if (hlen >= ENCODE_SIZE_WITH_COST_TIME) { obRpcCostTime.decode(buf); @@ -181,7 +245,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Ignore unresolved bytes. */ public void ignoreUnresolvedBytes(ByteBuf buf, int hlen, int encodeSize) { @@ -190,49 +254,49 @@ public void ignoreUnresolvedBytes(ByteBuf buf, int hlen, int encodeSize) { } } - /** + /* * Is response. */ public boolean isResponse() { return (flag & RESP_FLAG) != 0; } - /** + /* * Is stream. */ public boolean isStream() { return (flag & STREAM_FLAG) != 0; } - /** + /* * Is stream next. */ public boolean isStreamNext() { return isStream() && (flag & STREAM_LAST_FLAG) == 0; } - /** + /* * Is stream last. */ public boolean isStreamLast() { return isStream() && (flag & STREAM_LAST_FLAG) != 0; } - /** + /* * Is routing wrong. */ public boolean isRoutingWrong() { return (flag & REQUIRE_REROUTING_FLAG) != 0; } - /** + /* * Set routing wrong flag bit. */ public void setRoutingWrong() { flag |= REQUIRE_REROUTING_FLAG; } - /** + /* * Set stream next. */ public void setStreamNext() { @@ -240,7 +304,7 @@ public void setStreamNext() { flag |= STREAM_FLAG; } - /** + /* * Set stream last. */ public void setStreamLast() { @@ -248,224 +312,235 @@ public void setStreamLast() { flag |= STREAM_FLAG; } - /** + /* + * Set routing flag bit. + */ + public void enableRerouting() { + flag |= REQUIRE_REROUTING_FLAG; + } + + public void disableRerouting() { + flag &= ~REQUIRE_REROUTING_FLAG; + } + + /* * Get pcode. */ public int getPcode() { return pcode; } - /** + /* * Set pcode. */ public void setPcode(int pcode) { this.pcode = pcode; } - /** + /* * Get hlen. */ public short getHlen() { return hlen; } - /** + /* * Set hlen. */ public void setHlen(byte hlen) { this.hlen = hlen; } - /** + /* * Get priority. */ public short getPriority() { return priority; } - /** + /* * Set priority. */ public void setPriority(short priority) { this.priority = priority; } - /** + /* * Get flag. */ public short getFlag() { return flag; } - /** + /* * Set flag. */ public void setFlag(short flag) { this.flag = flag; } - /** + /* * Get checksum. */ public long getChecksum() { return checksum; } - /** + /* * Set checksum. */ public void setChecksum(long checksum) { this.checksum = checksum; } - /** + /* * Get tenant id. */ public long getTenantId() { return tenantId; } - /** + /* * Set tenant id. */ public void setTenantId(long tenantId) { this.tenantId = tenantId; } - /** + /* * Get prv tenant id. */ public long getPrvTenantId() { return prvTenantId; } - /** + /* * Set prv tenant id. */ public void setPrvTenantId(long prvTenantId) { this.prvTenantId = prvTenantId; } - /** + /* * Get session id. */ public long getSessionId() { return sessionId; } - /** + /* * Set session id. */ public void setSessionId(long sessionId) { this.sessionId = sessionId; } - /** + /* * Get trace id0. */ public long getTraceId0() { return traceId0; } - /** + /* * Set trace id0. */ public void setTraceId0(long traceId0) { this.traceId0 = traceId0; } - /** + /* * Get trace id1. */ public long getTraceId1() { return traceId1; } - /** + /* * Set trace id1. */ public void setTraceId1(long traceId1) { this.traceId1 = traceId1; } - /** + /* * Get timeout. */ public long getTimeout() { return timeout; } - /** + /* * Set timeout. */ public void setTimeout(long timeout) { this.timeout = timeout; } - /** + /* * Get timestamp. */ public long getTimestamp() { return timestamp; } - /** + /* * Set timestamp. */ public void setTimestamp(long timestamp) { this.timestamp = timestamp; } - /** + /* * Get ob rpc cost time. */ public ObRpcCostTime getObRpcCostTime() { return obRpcCostTime; } - /** + /* * Set ob rpc cost time. */ public void setObRpcCostTime(ObRpcCostTime obRpcCostTime) { this.obRpcCostTime = obRpcCostTime; } - /** + /* * Get cluster id. */ - public long getClusterId() { - return clusterId; + public long getDstClusterId() { + return dstClusterId; } - /** + /* * Set cluster id. */ - public void setClusterId(long clusterId) { - this.clusterId = clusterId; + public void setDstClusterId(long dstClusterId) { + this.dstClusterId = dstClusterId; } - /** + /* * Get ob compress type. */ public ObCompressType getObCompressType() { return obCompressType; } - /** + /* * Set ob compress type. */ public void setObCompressType(ObCompressType obCompressType) { this.obCompressType = obCompressType; } - /** + /* * Get original len. */ public int getOriginalLen() { return originalLen; } - /** + /* * Set original len. */ public void setOriginalLen(int originalLen) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/AbstractPayload.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/AbstractPayload.java index f7665396..a4e2322e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/AbstractPayload.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/AbstractPayload.java @@ -23,9 +23,10 @@ import java.util.concurrent.atomic.AtomicInteger; import static com.alipay.oceanbase.rpc.property.Property.RPC_OPERATION_TIMEOUT; +import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; -/** +/* * * varlong varlong plain bytes * ----------------------------------- @@ -43,7 +44,7 @@ public abstract class AbstractPayload implements ObPayload { private long version = 1; protected long timeout = RPC_OPERATION_TIMEOUT.getDefaultLong(); - /** + /* * Get pcode. */ @Override @@ -51,7 +52,7 @@ public int getPcode() { return Pcodes.OB_ERROR_PACKET; } - /** + /* * Get timeout. */ @Override @@ -59,7 +60,7 @@ public long getTimeout() { return timeout; } - /** + /* * Get version. */ @Override @@ -67,21 +68,21 @@ public long getVersion() { return version; } - /** + /* * Set version. */ public void setVersion(long version) { this.version = version; } - /** + /* * Set channel id. */ public void setChannelId(Integer channelId) { this.channelId = channelId; } - /** + /* * Get payload size. */ @Override @@ -90,7 +91,7 @@ public long getPayloadSize() { return getObUniVersionHeaderLength(getVersion(), payloadContentSize) + payloadContentSize; } - /** + /* * Get channel id. */ @Override @@ -101,7 +102,7 @@ public int getChannelId() { return channelId; } - /** + /* * Get tenant id. */ @Override @@ -109,14 +110,14 @@ public long getTenantId() { return tenantId; } - /** + /* * Set tenant id. */ public void setTenantId(long tenantId) { this.tenantId = tenantId; } - /** + /* * Decode. */ @Override @@ -127,35 +128,43 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get unique id. */ - @Override public long getUniqueId() { return uniqueId; } - /** + /* * Set unique id. */ - @Override public void setUniqueId(long uniqueId) { this.uniqueId = uniqueId; } - /** + /* * Get sequence. */ - @Override public long getSequence() { return sequence; } - /** + /* * Set sequence. */ - @Override public void setSequence(long sequence) { this.sequence = sequence; } + + /* + * encode unis header + */ + protected int encodeHeader(byte[] bytes, int idx) { + int headerLen = (int) getObUniVersionHeaderLength(getVersion(), getPayloadContentSize()); + System.arraycopy(encodeObUniVersionHeader(getVersion(), getPayloadContentSize()), 0, bytes, + idx, headerLen); + idx += headerLen; + return idx; + } + } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Constants.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Constants.java index 27690cae..26ecc1cc 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Constants.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Constants.java @@ -21,12 +21,16 @@ public interface Constants { long OB_INVALID_ID = -1; - short UNSIGNED_INT8_MAX = 1 << 8 - 1; + long INVALID_TABLET_ID = 0; - int UNSIGNED_INT16_MAX = 1 << 16 - 1; + short UNSIGNED_INT8_MAX = (1 << 8) - 1; - int UNSIGNED_INT24_MAX = 1 << 24 - 1; + int UNSIGNED_INT16_MAX = (1 << 16) - 1; - long UNSIGNED_INT32_MAX = 1L << 32 - 1; + int UNSIGNED_INT24_MAX = (1 << 24) - 1; + + long UNSIGNED_INT32_MAX = (1L << 32) - 1; + + long INVALID_LS_ID = -1; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObPayload.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObPayload.java index 3bbf640f..c1bd418b 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObPayload.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObPayload.java @@ -21,32 +21,52 @@ public interface ObPayload extends ObUnisVersion { - /** + /* * @return channel id */ int getChannelId(); - /** + /* * @return tenant id */ long getTenantId(); - /** + /* * @return protocol code */ int getPcode(); - /** + /* * @return timeout */ long getTimeout(); - /** + /* + * set sequence + */ + void setSequence(long sequence); + + /* + * @return sequence + */ + long getSequence(); + + /* + * set unique id + */ + void setUniqueId(long uniqueId); + + /* + * @return unique id + */ + long getUniqueId(); + + /* * @return encoded payload content */ byte[] encode(); - /** + /* * Decode payload from byte buffer * * @param buf buf from net framework @@ -54,12 +74,12 @@ public interface ObPayload extends ObUnisVersion { */ Object decode(ByteBuf buf); - /** + /* * @return payload size, include header bytes */ long getPayloadSize(); - /** + /* * @return payload content size, without header bytes */ long getPayloadContentSize(); diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObRpcResultCode.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObRpcResultCode.java index 19d883b3..34b1b466 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObRpcResultCode.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObRpcResultCode.java @@ -32,13 +32,13 @@ public class ObRpcResultCode extends AbstractPayload { private byte[] msg = new byte[0]; private List warningMsgs = new ArrayList(); - /** + /* * Ob rpc result code. */ public ObRpcResultCode() { } - /** + /* * Encode. */ public byte[] encode() { @@ -72,7 +72,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -95,7 +95,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -110,42 +110,52 @@ public long getPayloadContentSize() { return size; } - /** + /* * Get rcode. */ public int getRcode() { return rcode; } - /** + /* * Set rcode. */ public void setRcode(int rcode) { this.rcode = rcode; } - /** + /* * Get msg. */ public byte[] getMsg() { return msg; } - /** + /* + * Get error msg. + */ + public String getErrMsg() { + if (msg != null && msg.length > 0) { + return new String(msg, 0, msg.length - 1); + } + return new String(msg); + } + + /* * Set msg. */ public void setMsg(byte[] msg) { this.msg = msg; } - /** + /* * Get warning msgs. */ public List getWarningMsgs() { return warningMsgs; } - /** + /* * Set warning msgs. */ public void setWarningMsgs(List warningMsgs) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObRpcResultWarningMsg.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObRpcResultWarningMsg.java index 8465b923..83124e46 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObRpcResultWarningMsg.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObRpcResultWarningMsg.java @@ -31,13 +31,13 @@ public class ObRpcResultWarningMsg extends AbstractPayload { private int lineNo; private int code; - /** + /* * Ob rpc result warning msg. */ public ObRpcResultWarningMsg() { } - /** + /* * Get pcode. */ @Override @@ -45,7 +45,7 @@ public int getPcode() { return 0; } - /** + /* * Encode. */ @Override @@ -78,7 +78,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -97,7 +97,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -107,70 +107,70 @@ public long getPayloadContentSize() { + Serialization.getNeedBytes(lineNo) + Serialization.getNeedBytes(code); } - /** + /* * Get msg. */ public byte[] getMsg() { return msg; } - /** + /* * Set msg. */ public void setMsg(byte[] msg) { this.msg = msg; } - /** + /* * Get timestamp. */ public long getTimestamp() { return timestamp; } - /** + /* * Set timestamp. */ public void setTimestamp(long timestamp) { this.timestamp = timestamp; } - /** + /* * Get log level. */ public int getLogLevel() { return logLevel; } - /** + /* * Set log level. */ public void setLogLevel(int logLevel) { this.logLevel = logLevel; } - /** + /* * Get line no. */ public int getLineNo() { return lineNo; } - /** + /* * Set line no. */ public void setLineNo(int lineNo) { this.lineNo = lineNo; } - /** + /* * Get code. */ public int getCode() { return code; } - /** + /* * Set code. */ public void setCode(int code) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObSimplePayload.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObSimplePayload.java index 154a3159..735cd9fc 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObSimplePayload.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObSimplePayload.java @@ -17,16 +17,19 @@ package com.alipay.oceanbase.rpc.protocol.payload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; + import io.netty.buffer.ByteBuf; -/** +/* * Ob Simple payload, only need encode & decode methods - * */ public interface ObSimplePayload { byte[] encode(); + void encode(ObByteBuf buf); + Object decode(ByteBuf buf); int getEncodedSize(); diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObUnisVersion.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObUnisVersion.java index 83e3e69d..50e12349 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObUnisVersion.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ObUnisVersion.java @@ -21,11 +21,4 @@ public interface ObUnisVersion { long getVersion(); - void setSequence(long sequence); - - long getSequence(); - - void setUniqueId(long uniqueId); - - long getUniqueId(); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Pcodes.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Pcodes.java index ffa31066..5eec03e2 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Pcodes.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/Pcodes.java @@ -19,13 +19,18 @@ public interface Pcodes { - int OB_TABLE_API_LOGIN = 0x1101; - int OB_TABLE_API_EXECUTE = 0x1102; - int OB_TABLE_API_BATCH_EXECUTE = 0x1103; - int OB_TABLE_API_EXECUTE_QUERY = 0x1104; - int OB_TABLE_API_QUERY_AND_MUTATE = 0x1105; + int OB_TABLE_API_LOGIN = 0x1101; + int OB_TABLE_API_EXECUTE = 0x1102; + int OB_TABLE_API_BATCH_EXECUTE = 0x1103; + int OB_TABLE_API_EXECUTE_QUERY = 0x1104; + int OB_TABLE_API_QUERY_AND_MUTATE = 0x1105; // INVALID REQUEST PCODE, no such rpc - int OB_ERROR_PACKET = 0x010; + int OB_ERROR_PACKET = 0x010; + int OB_TABLE_API_EXECUTE_QUERY_SYNC = 0x1106; + + int OB_TABLE_API_DIRECT_LOAD = 0x1123; + int OB_TABLE_API_MOVE = 0x1124; + int OB_TABLE_API_LS_EXECUTE = 0x1125; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java index 1031e38d..fdfef301 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/ResultCodes.java @@ -70,6 +70,7 @@ public enum ResultCodes { OB_NEED_RETRY(-4036), // OB_TOO_MANY_SSTABLE(-4037), // OB_NOT_MASTER(-4038), // + OB_KILLED_BY_THROTTLING(-4039), // OB_DECRYPT_FAILED(-4041), // OB_USER_NOT_EXIST(-4042), // OB_PASSWORD_WRONG(-4043), // @@ -357,6 +358,9 @@ public enum ResultCodes { OB_CLUSTER_NO_MATCH(-4666), // OB_CHECK_ZONE_MERGE_ORDER(-4667), // OB_ERR_ZONE_NOT_EMPTY(-4668), // + OB_USE_DUP_FOLLOW_AFTER_DML(-4686), + OB_LS_NOT_EXIST(-4719), // + OB_TABLET_NOT_EXIST(-4725), // OB_ERR_PARSER_INIT(-5000), // OB_ERR_PARSE_SQL(-5001), // OB_ERR_RESOLVE_SQL(-5002), // @@ -631,6 +635,7 @@ public enum ResultCodes { OB_IGNORE_SQL_IN_RESTORE(-5309), // OB_ERR_UNEXPECTED_TZ_TRANSITION(-5310), // OB_ERR_INVALID_COLUMN_ID(-5311), // + OB_SCHEMA_EAGAIN(-5627), // OB_TRANSACTION_SET_VIOLATION(-6001), // OB_TRANS_ROLLBACKED(-6002), // OB_ERR_EXCLUSIVE_LOCK_CONFLICT(-6003), // @@ -663,6 +668,7 @@ public enum ResultCodes { OB_PARTITION_IS_BLOCKED(-6229), // OB_TRANS_RPC_TIMEOUT(-6230), // OB_REPLICA_NOT_READABLE(-6231), // + OB_TRANS_STMT_NEED_RETRY(-6241), // OB_LOG_ID_NOT_FOUND(-6301), // OB_LSR_THREAD_STOPPED(-6302), // OB_NO_LOG(-6303), // @@ -721,7 +727,17 @@ public enum ResultCodes { OB_AGENT_INITING_BACKUP_COUNT_ERROR(-9015), // OB_CLUSTER_NAME_NOT_EQUAL(-9016), // OB_RS_LIST_INVAILD(-9017), // - OB_AGENT_HAS_FAILED_TASK(-9018); + OB_AGENT_HAS_FAILED_TASK(-9018), // + OB_ERR_KV_GLOBAL_INDEX_ROUTE(-10500), // + OB_KV_CREDENTIAL_NOT_MATCH(-10509), // + OB_KV_ROWKEY_COUNT_NOT_MATCH(-10510), // + OB_KV_COLUMN_TYPE_NOT_MATCH(-10511), // + OB_KV_COLLATION_MISMATCH(-10512), // + OB_KV_SCAN_RANGE_MISSING(-10513), // + OB_KV_FILTER_PARSE_ERROR(-10514), // + OB_KV_REDIS_PARSE_ERROR(-10515), // + OB_KV_HBASE_INCR_FIELD_IS_NOT_LONG(-10516), // + OB_KV_ODP_TIMEOUT(-10650); public final int errorCode; @@ -737,7 +753,7 @@ public enum ResultCodes { } } - /** + /* * Value of. */ public static ResultCodes valueOf(int errorCode) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObAddr.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObAddr.java new file mode 100644 index 00000000..5e18df09 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObAddr.java @@ -0,0 +1,150 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl; + +import java.util.HashMap; +import java.util.Map; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +public class ObAddr /*extends ObUnisVersion*/implements ObSimplePayload { + + private static final long UNIS_VERSION = 1; + + private VER version = VER.IPV4; + private int[] ip = new int[4]; + private int port = 0; + + public ObAddr() { + } + + public VER getVersion() { + return version; + } + + public String getIPString() { + String ipStr = new String(); + if (version == VER.IPV4) { + ipStr = String.format("%d.%d.%d.%d", (ip[0] >> 24) & 0XFF, (ip[0] >> 16) & 0XFF, + (ip[0] >> 8) & 0XFF, (ip[0]) & 0XFF); + } + return ipStr; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String toString() { + String str = new String(); + if (version == VER.IPV4) { + str = String.format("%s:%d", getIPString(), port); + } + return str; + } + + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + @Override + public void encode(ObByteBuf buf) { + long payloadContentSize = getPayloadContentSize(); + Serialization.encodeObUniVersionHeader(buf, UNIS_VERSION, payloadContentSize); + Serialization.encodeI8(buf, version.getByteValue()); + for (int i = 0; i < ip.length; ++i) { + Serialization.encodeVi32(buf, ip[i]); + } + Serialization.encodeVi32(buf, port); + } + + @Override + public Object decode(ByteBuf buf) { + long unis_version = Serialization.decodeVi64(buf); + if (unis_version != UNIS_VERSION) { + throw new ObTableException("object version mismatch, version:" + unis_version); + } + Serialization.decodeVi64(buf); // get payload length, useless now + version = VER.valueOf(Serialization.decodeI8(buf)); + for (int i = 0; i < ip.length; ++i) { + ip[i] = Serialization.decodeVi32(buf); + } + port = Serialization.decodeVi32(buf); + return this; + } + + public long getPayloadContentSize() { + long payloadContentSize = 0; + payloadContentSize += 1; + for (int i = 0; i < ip.length; ++i) { + payloadContentSize += Serialization.getNeedBytes(ip[i]); + } + payloadContentSize += Serialization.getNeedBytes(port); + return payloadContentSize; + } + + @Override + public int getEncodedSize() { + long payloadContentSize = getPayloadContentSize(); + return (int) (Serialization.getObUniVersionHeaderLength(UNIS_VERSION, payloadContentSize) + payloadContentSize); + } + + public enum VER { + + IPV4(4), IPV6(6), UNIX(1); + + private final int value; + private static final Map map = new HashMap(); + + static { + for (VER v : VER.values()) { + map.put(v.value, v); + } + } + + public static VER valueOf(int value) { + return map.get(value); + } + + VER(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + + public byte getByteValue() { + return (byte) value; + } + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObCollationLevel.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObCollationLevel.java index 41e9ca92..bf5eb272 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObCollationLevel.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObCollationLevel.java @@ -48,21 +48,21 @@ public enum ObCollationLevel { } } - /** + /* * Value of. */ public static ObCollationLevel valueOf(int value) { return map.get(value); } - /** + /* * Get value. */ public int getValue() { return value; } - /** + /* * Get byte value. */ public byte getByteValue() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObCollationType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObCollationType.java index b4b852ff..9dd2fe01 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObCollationType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObCollationType.java @@ -42,21 +42,21 @@ public enum ObCollationType { } } - /** + /* * Value of. */ public static ObCollationType valueOf(int value) { return map.get(value); } - /** + /* * Get value. */ public int getValue() { return value; } - /** + /* * Get byte value. */ public byte getByteValue() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObColumn.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObColumn.java index 40b8f47a..4df579fd 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObColumn.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObColumn.java @@ -30,7 +30,7 @@ public abstract class ObColumn { protected final List refColumnNames; protected final ObGeneratedColumnSimpleFunc columnExpress; - /** + /* * Ob column. */ public ObColumn(String columnName, int index, ObObjType obObjType, @@ -44,42 +44,42 @@ public ObColumn(String columnName, int index, ObObjType obObjType, this.columnExpress = columnExpress; } - /** + /* * Get column name. */ public String getColumnName() { return columnName; } - /** + /* * Get index. */ public int getIndex() { return index; } - /** + /* * Get ob obj type. */ public ObObjType getObObjType() { return obObjType; } - /** + /* * Get ob collation type. */ public ObCollationType getObCollationType() { return obCollationType; } - /** + /* * Get ref column names. */ public List getRefColumnNames() { return refColumnNames; } - /** + /* * Get ob generated column simple func. */ public ObGeneratedColumnSimpleFunc getObGeneratedColumnSimpleFunc() { @@ -88,7 +88,7 @@ public ObGeneratedColumnSimpleFunc getObGeneratedColumnSimpleFunc() { public abstract Object evalValue(Object... refs) throws IllegalArgumentException; - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObj.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObj.java index 4320b4a0..7fa4e1b5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObj.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObj.java @@ -18,6 +18,7 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl; import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; import com.alipay.oceanbase.rpc.util.ObVString; import io.netty.buffer.ByteBuf; @@ -25,12 +26,15 @@ public class ObObj implements ObSimplePayload { private static ObObj MAX_OBJECT; private static ObObj MIN_OBJECT; + private static long MAX_OBJECT_VALUE = -2L; + private static long MIN_OBJECT_VALUE = -3L; + static { - MAX_OBJECT = new ObObj(ObObjType.ObExtendType.getDefaultObjMeta(), -2L); - MIN_OBJECT = new ObObj(ObObjType.ObExtendType.getDefaultObjMeta(), -3L); + MAX_OBJECT = new ObObj(ObObjType.ObExtendType.getDefaultObjMeta(), MAX_OBJECT_VALUE); + MIN_OBJECT = new ObObj(ObObjType.ObExtendType.getDefaultObjMeta(), MIN_OBJECT_VALUE); } - /** + /* * Ob obj. */ public ObObj() { @@ -38,7 +42,7 @@ public ObObj() { this.value = null; } - /** + /* * Ob obj. */ public ObObj(ObObjMeta meta, Object value) { @@ -50,7 +54,7 @@ public ObObj(ObObjMeta meta, Object value) { private ObObjMeta meta; // 2. 序列化 valueLength 还是 nmb_desc_？ private int valueLength; - /** + /* *3. 序列化 value。 * obj_val_serialize, * obj_val_deserialize, @@ -58,7 +62,7 @@ public ObObj(ObObjMeta meta, Object value) { */ private Object value; - /** + /* * Encode. */ @Override @@ -68,12 +72,24 @@ public byte[] encode() { System.arraycopy(meta.encode(), 0, bytes, idx, 4); idx += 4; - System.arraycopy(meta.getType().encode(value), 0, bytes, idx, meta.getType() - .getEncodedSize(value)); + + byte[] valueBytes = meta.getType().encode(value); + System.arraycopy(valueBytes, 0, bytes, idx, valueBytes.length); + idx += valueBytes.length; + return bytes; } - /** + /* + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + meta.encode(buf); + buf.writeBytes(meta.getType().encode(value)); + } + + /* * Decode. */ @Override @@ -84,7 +100,7 @@ public ObObj decode(ByteBuf buf) { return this; } - /** + /* * Get encoded size. */ @Override @@ -92,42 +108,42 @@ public int getEncodedSize() { return meta.getEncodedSize() + meta.getType().getEncodedSize(value); } - /** + /* * Get meta. */ public ObObjMeta getMeta() { return meta; } - /** + /* * Set meta. */ public void setMeta(ObObjMeta meta) { this.meta = meta; } - /** + /* * Get value length. */ public int getValueLength() { return valueLength; } - /** + /* * Set value length. */ public void setValueLength(int valueLength) { this.valueLength = valueLength; } - /** + /* * Get value. */ public Object getValue() { return value; } - /** + /* * Set value. */ public void setValue(Object value) { @@ -138,29 +154,33 @@ public void setValue(Object value) { } } - /** + /* * Get instance. */ public static ObObj getInstance(Object value) { ObObjMeta meta = ObObjType.defaultObjMeta(value); - return new ObObj(meta, value); + if (value instanceof ObObj) { + return new ObObj(meta, ((ObObj) value).getValue()); + } else { + return new ObObj(meta, value); + } } - /** + /* * Get max. */ public static ObObj getMax() { return MAX_OBJECT; } - /** + /* * Get min. */ public static ObObj getMin() { return MIN_OBJECT; } - /** + /* * To string. */ @Override @@ -168,4 +188,19 @@ public String toString() { return "ObObj{" + "meta=" + meta + ", valueLength=" + valueLength + ", value=" + value + '}'; } + + public boolean isMinObj() { + return (getMeta().getType() == ObObjType.ObExtendType) + && ((long) getValue() == MIN_OBJECT_VALUE); + } + + public boolean isMaxObj() { + return (getMeta().getType() == ObObjType.ObExtendType) + && ((long) getValue() == MAX_OBJECT_VALUE); + } + + // set value directly, do not wrapped by ObVString when type is varchar + public void setValueFromTableObj(Object value) { + this.value = value; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObjMeta.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObjMeta.java index a9eda08a..ef71e3e1 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObjMeta.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObjMeta.java @@ -18,6 +18,7 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl; import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; @@ -62,13 +63,13 @@ public class ObObjMeta implements ObSimplePayload { private ObCollationType csType; // collation type QUESTION: ObCollationType 定义 private byte scale; // scale, 当type_ 为ObBitType时，该字段存储bit的length - /** + /* * Ob obj meta. */ public ObObjMeta() { } - /** + /* * Ob obj meta. */ public ObObjMeta(ObObjType type, ObCollationLevel csLevel, ObCollationType csType, byte scale) { @@ -78,7 +79,7 @@ public ObObjMeta(ObObjType type, ObCollationLevel csLevel, ObCollationType csTyp this.scale = scale; } - /** + /* * Encode. */ @Override @@ -97,6 +98,17 @@ public byte[] encode() { } /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeI8(buf, type.getByteValue()); + Serialization.encodeI8(buf, csLevel.getByteValue()); + Serialization.encodeI8(buf, csType.getByteValue()); + Serialization.encodeI8(buf, scale); + } + + /* * Decode. */ @Override @@ -109,7 +121,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get encoded size. */ @Override @@ -117,63 +129,63 @@ public int getEncodedSize() { return 4; } - /** + /* * Get type. */ public ObObjType getType() { return type; } - /** + /* * Set type. */ public void setType(ObObjType type) { this.type = type; } - /** + /* * Get cs level. */ public ObCollationLevel getCsLevel() { return csLevel; } - /** + /* * Set cs level. */ public void setCsLevel(ObCollationLevel csLevel) { this.csLevel = csLevel; } - /** + /* * Get cs type. */ public ObCollationType getCsType() { return csType; } - /** + /* * Set cs type. */ public void setCsType(ObCollationType csType) { this.csType = csType; } - /** + /* * Get scale. */ public byte getScale() { return scale; } - /** + /* * Set scale. */ public void setScale(byte scale) { this.scale = scale; } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObjType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObjType.java index 216a9381..b100b61d 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObjType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObObjType.java @@ -26,14 +26,13 @@ import io.netty.buffer.ByteBuf; import java.sql.Timestamp; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; +import java.time.*; +import java.util.*; public enum ObObjType { ObNullType(0) { - /** + /* * Encode. */ @Override @@ -41,7 +40,7 @@ public byte[] encode(Object obj) { return new byte[0]; } - /** + /* * Decode. */ @Override @@ -49,7 +48,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return null; } - /** + /* * Get encoded size. */ @Override @@ -57,7 +56,7 @@ public int getEncodedSize(Object obj) { return 0; } - /** + /* * Get default obj meta. */ @Override @@ -66,7 +65,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -79,7 +78,7 @@ public Comparable parseToComparable(Object o, ObCollationType ct) }, // 空类型 ObTinyIntType(1) { - /** + /* * Encode. */ @Override @@ -91,7 +90,7 @@ public byte[] encode(Object obj) { } } - /** + /* * Decode. */ @Override @@ -99,7 +98,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeI8(buf.readByte()); } - /** + /* * Get encoded size. */ @Override @@ -107,7 +106,7 @@ public int getEncodedSize(Object obj) { return 1; } - /** + /* * Get default obj meta. */ @Override @@ -116,7 +115,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -134,7 +133,7 @@ public Byte parseToComparable(Object o, ObCollationType ct) } }, // int8, aka mysql boolean type ObSmallIntType(2) { - /** + /* * Encode. */ @Override @@ -142,7 +141,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi32(((Number) obj).intValue()); } - /** + /* * Decode. */ @Override @@ -150,7 +149,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return (short) Serialization.decodeVi32(buf); } - /** + /* * Get encoded size. */ @Override @@ -158,7 +157,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Number) obj).intValue()); } - /** + /* * Get default obj meta. */ @Override @@ -167,7 +166,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -188,7 +187,7 @@ public Short parseToComparable(Object o, ObCollationType ct) @Deprecated ObMediumIntType(3) { // // TODO not suppor - /** + /* * Encode. */ @Override @@ -196,7 +195,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi32(((Number) obj).intValue()); } - /** + /* * Decode. */ @Override @@ -204,7 +203,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeVi32(buf); } - /** + /* * Get encoded size. */ @Override @@ -212,7 +211,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Number) obj).intValue()); } - /** + /* * Get default obj meta. */ @Override @@ -221,7 +220,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -232,7 +231,7 @@ public Integer parseToComparable(Object o, ObCollationType ct) } }, // int24 ObInt32Type(4) { - /** + /* * Encode. */ @Override @@ -240,7 +239,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi32(((Number) obj).intValue()); } - /** + /* * Decode. */ @Override @@ -248,7 +247,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeVi32(buf); } - /** + /* * Get encoded size. */ @Override @@ -256,7 +255,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Number) obj).intValue()); } - /** + /* * Get default obj meta. */ @Override @@ -265,7 +264,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -284,7 +283,7 @@ public Integer parseToComparable(Object o, ObCollationType ct) }, // int32 ObInt64Type(5) { // Origin name: ObIntType - /** + /* * Encode. */ @Override @@ -292,7 +291,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi64(((Number) obj).longValue()); } - /** + /* * Decode. */ @Override @@ -300,7 +299,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeVi64(buf); } - /** + /* * Get encoded size. */ @Override @@ -308,7 +307,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Number) obj).longValue()); } - /** + /* * Get default obj meta. */ @Override @@ -317,7 +316,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -329,7 +328,7 @@ public Long parseToComparable(Object o, ObCollationType ct) }, // int64, aka bigint ObUTinyIntType(6) { - /** + /* * Encode. */ @Override @@ -337,7 +336,7 @@ public byte[] encode(Object obj) { return new byte[] { (Byte) obj }; } - /** + /* * Decode. */ @Override @@ -345,7 +344,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeI8(buf.readByte()); } - /** + /* * Get encoded size. */ @Override @@ -353,7 +352,7 @@ public int getEncodedSize(Object obj) { return 1; } - /** + /* * Get default obj meta. */ @Override @@ -362,7 +361,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -381,7 +380,7 @@ public Short parseToComparable(Object o, ObCollationType ct) }, // uint8 ObUSmallIntType(7) { // TODO not support - /** + /* * Encode. */ @Override @@ -389,7 +388,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi32(((Number) obj).intValue()); } - /** + /* * Decode. */ @Override @@ -397,7 +396,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeVi32(buf); } - /** + /* * Get encoded size. */ @Override @@ -405,7 +404,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Number) obj).intValue()); } - /** + /* * Get default obj meta. */ @Override @@ -414,7 +413,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -435,7 +434,7 @@ public Integer parseToComparable(Object o, ObCollationType ct) }, // uint16 ObUMediumIntType(8) { // TODO not support - /** + /* * Encode. */ @Override @@ -443,7 +442,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi32(((Number) obj).intValue()); } - /** + /* * Decode. */ @Override @@ -451,7 +450,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeVi32(buf); } - /** + /* * Get encoded size. */ @Override @@ -459,7 +458,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Number) obj).intValue()); } - /** + /* * Get default obj meta. */ @Override @@ -468,7 +467,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -489,7 +488,7 @@ public Integer parseToComparable(Object o, ObCollationType ct) }, // uint24 ObUInt32Type(9) { // TODO not support - /** + /* * Encode. */ @Override @@ -497,7 +496,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi32(((Number) obj).intValue()); } - /** + /* * Decode. */ @Override @@ -505,7 +504,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeVi32(buf); } - /** + /* * Get encoded size. */ @Override @@ -513,16 +512,16 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Number) obj).intValue()); } - /** + /* * Get default obj meta. */ @Override public ObObjMeta getDefaultObjMeta() { return new ObObjMeta(this, ObCollationLevel.CS_LEVEL_NUMERIC, - ObCollationType.CS_TYPE_BINARY, (byte) 10); + ObCollationType.CS_TYPE_BINARY, (byte) -1); } - /** + /* * Parse to comparable. */ @Override @@ -542,7 +541,7 @@ public Long parseToComparable(Object o, ObCollationType ct) }, // uint32 ObUInt64Type(10) { // TODO not support - /** + /* * Encode. */ @Override @@ -551,7 +550,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi64(((Number) obj).longValue()); } - /** + /* * Decode. */ @Override @@ -560,7 +559,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeVi64(buf); } - /** + /* * Get encoded size. */ @Override @@ -568,7 +567,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Number) obj).longValue()); } - /** + /* * Get default obj meta. */ @Override @@ -577,7 +576,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -589,7 +588,7 @@ public Long parseToComparable(Object o, ObCollationType ct) }, // uint64 ObFloatType(11) { - /** + /* * Encode. */ @Override @@ -597,7 +596,7 @@ public byte[] encode(Object obj) { return Serialization.encodeFloat(((Float) obj)); } - /** + /* * Decode. */ @Override @@ -605,7 +604,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeFloat(buf); } - /** + /* * Get encoded size. */ @Override @@ -613,7 +612,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes((Float) obj); } - /** + /* * Get default obj meta. */ @Override @@ -622,7 +621,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -646,7 +645,7 @@ public Float parseToComparable(Object o, ObCollationType ct) } }, // single-precision floating point ObDoubleType(12) { - /** + /* * Encode. */ @Override @@ -654,7 +653,7 @@ public byte[] encode(Object obj) { return Serialization.encodeDouble((Double) obj); } - /** + /* * Decode. */ @Override @@ -662,7 +661,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeDouble(buf); } - /** + /* * Get encoded size. */ @Override @@ -670,7 +669,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes((Double) obj); } - /** + /* * Get default obj meta. */ @Override @@ -679,7 +678,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -705,7 +704,7 @@ public Double parseToComparable(Object o, ObCollationType ct) ObUFloatType(13) { // TODO not support - /** + /* * Encode. */ @Override @@ -713,7 +712,7 @@ public byte[] encode(Object obj) { return new byte[0]; } - /** + /* * Decode. */ @Override @@ -721,7 +720,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return null; } - /** + /* * Get encoded size. */ @Override @@ -729,7 +728,7 @@ public int getEncodedSize(Object obj) { return 0; } - /** + /* * Get default obj meta. */ @Override @@ -738,7 +737,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -750,7 +749,7 @@ public Float parseToComparable(Object o, ObCollationType ct) }, // unsigned single-precision floating point ObUDoubleType(14) { // TODO not support - /** + /* * Encode. */ @Override @@ -758,7 +757,7 @@ public byte[] encode(Object obj) { return new byte[0]; } - /** + /* * Decode. */ @Override @@ -766,7 +765,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return null; } - /** + /* * Get encoded size. */ @Override @@ -774,7 +773,7 @@ public int getEncodedSize(Object obj) { return 0; } - /** + /* * Get default obj meta. */ @Override @@ -783,7 +782,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -796,7 +795,7 @@ public Double parseToComparable(Object o, ObCollationType ct) ObNumberType(15) { // TODO - /** + /* * Encode. */ @Override @@ -804,7 +803,7 @@ public byte[] encode(Object obj) { return new byte[0]; } - /** + /* * Decode. */ @Override @@ -812,7 +811,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return null; } - /** + /* * Get encoded size. */ @Override @@ -820,7 +819,7 @@ public int getEncodedSize(Object obj) { return 0; } - /** + /* * Get default obj meta. */ @Override @@ -829,7 +828,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -841,7 +840,7 @@ public Comparable parseToComparable(Object o, ObCollationType ct) }, // aka decimal/numeric ObUNumberType(16) { // TODO not support - /** + /* * Encode. */ @Override @@ -849,7 +848,7 @@ public byte[] encode(Object obj) { return new byte[0]; } - /** + /* * Decode. */ @Override @@ -857,7 +856,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return null; } - /** + /* * Get encoded size. */ @Override @@ -865,7 +864,7 @@ public int getEncodedSize(Object obj) { return 0; } - /** + /* * Get default obj meta. */ @Override @@ -874,7 +873,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -885,27 +884,31 @@ public Comparable parseToComparable(Object o, ObCollationType ct) } }, - // https://dev.mysql.com/doc/refman/5.7/en/datetime.html // The DATETIME type is used for values that contain both date and time parts. // MySQL retrieves and displays DATETIME values in 'YYYY-MM-DD HH:MM:SS' format. The supported range is '1000-01-01 00:00:00' to '9999-12-31 23:59:59'. ObDateTimeType(17) { - /** + /* * Encode. */ @Override public byte[] encode(Object obj) { - return Serialization.encodeVi64(((Date) obj).getTime() * 1000L); + // Date do not have timezone, when we use getTime, system will recognize it as our system timezone and transform it into UTC Time, which will changed the time. + // We should add back the lose part. + long targetTs = ((Date) obj).getTime() + + OffsetDateTime.now().getOffset().getTotalSeconds() * 1000L; + return Serialization.encodeVi64(targetTs * 1000L); } - /** + /* * Decode. */ @Override public Object decode(ByteBuf buf, ObCollationType type) { - return new Timestamp(Serialization.decodeVi64(buf) / 1000L); + return new Date(Serialization.decodeVi64(buf) / 1000L + - OffsetDateTime.now().getOffset().getTotalSeconds() * 1000L); } - /** + /* * Get encoded size. */ @Override @@ -913,46 +916,57 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Date) obj).getTime() * 1000L); } - /** + /* * Get default obj meta. */ @Override public ObObjMeta getDefaultObjMeta() { + // scale set into 6 means microSecond return new ObObjMeta(this, ObCollationLevel.CS_LEVEL_NUMERIC, - ObCollationType.CS_TYPE_BINARY, (byte) 10); + ObCollationType.CS_TYPE_BINARY, (byte) 6); } - /** + /* * Parse to comparable. */ @Override - public Timestamp parseToComparable(Object o, ObCollationType ct) - throws IllegalArgumentException, - FeatureNotSupportedException { - return parseTimestamp(this, o, ct); + public Date parseToComparable(Object o, ObCollationType ct) + throws IllegalArgumentException, + FeatureNotSupportedException { + if (o instanceof String) { + return TimeUtils.strToDate((String) o); + } + return (Date) o; } }, // The TIMESTAMP data type is used for values that contain both date and time parts. // TIMESTAMP has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC. ObTimestampType(18) { - /** + /* * Encode. */ @Override public byte[] encode(Object obj) { - // TODO 为什么是 Long - return Serialization.encodeVi64(((Date) obj).getTime() * 1000L); + long timeInMicroseconds = ((Timestamp)obj).getTime() * 1_000; + int nanoSeconds = ((Timestamp)obj).getNanos() % 1_000_000; + timeInMicroseconds += nanoSeconds / 1_000; + return Serialization.encodeVi64(timeInMicroseconds); } - /** + /* * Decode. */ @Override public Object decode(ByteBuf buf, ObCollationType type) { - return new Timestamp(Serialization.decodeVi64(buf) / 1000L); + long timestampMicro = Serialization.decodeVi64(buf); + long timestampMilli = timestampMicro / 1_000; + int nanos = (int) (timestampMicro % 1_000) * 1_000; + Timestamp timestamp = new Timestamp(timestampMilli); + timestamp.setNanos(timestamp.getNanos() + nanos); + return timestamp; } - /** + /* * Get encoded size. */ @Override @@ -960,16 +974,17 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Date) obj).getTime() * 1000L); } - /** + /* * Get default obj meta. */ @Override public ObObjMeta getDefaultObjMeta() { + // scale set into 6 means microSecond return new ObObjMeta(this, ObCollationLevel.CS_LEVEL_NUMERIC, - ObCollationType.CS_TYPE_BINARY, (byte) 10); + ObCollationType.CS_TYPE_BINARY, (byte) 6); } - /** + /* * Parse to comparable. */ @Override @@ -982,16 +997,15 @@ public Timestamp parseToComparable(Object o, ObCollationType ct) // The DATE type is used for values with a date part but no time part. // MySQL retrieves and displays DATE values in 'YYYY-MM-DD' format. The supported range is '1000-01-01' to '9999-12-31'. ObDateType(19) { - /** + /* * Encode. */ @Override public byte[] encode(Object obj) { - // TODO 为什么是 Int return Serialization.encodeVi32((int) ((Date) obj).getTime()); } - /** + /* * Decode. */ @Override @@ -999,7 +1013,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return new Date(Serialization.decodeVi32(buf) * 1000L); } - /** + /* * Get encoded size. */ @Override @@ -1007,7 +1021,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes((int) ((Date) obj).getTime()); } - /** + /* * Get default obj meta. */ @Override @@ -1016,7 +1030,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1045,7 +1059,7 @@ public Date parseToComparable(Object o, ObCollationType ct) }, ObTimeType(20) { // TODO not support - /** + /* * Encode. */ @Override @@ -1053,7 +1067,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi64((int) ((Date) obj).getTime()); } - /** + /* * Decode. */ @Override @@ -1061,7 +1075,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return new Timestamp(Serialization.decodeVi64(buf)); } - /** + /* * Get encoded size. */ @Override @@ -1069,7 +1083,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes((int) ((Date) obj).getTime()); } - /** + /* * Get default obj meta. */ @Override @@ -1078,7 +1092,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1090,7 +1104,7 @@ public Timestamp parseToComparable(Object o, ObCollationType ct) }, ObYearType(21) { // TODO not support - /** + /* * Encode. */ @Override @@ -1098,7 +1112,7 @@ public byte[] encode(Object obj) { return new byte[] { (Byte) obj }; } - /** + /* * Decode. */ @Override @@ -1106,7 +1120,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeI8(buf.readByte()); } - /** + /* * Get encoded size. */ @Override @@ -1114,7 +1128,7 @@ public int getEncodedSize(Object obj) { return 1; } - /** + /* * Get default obj meta. */ @Override @@ -1123,7 +1137,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1135,7 +1149,7 @@ public Comparable parseToComparable(Object o, ObCollationType ct) }, ObVarcharType(22) { - /** + /* * Encode. */ @Override @@ -1150,7 +1164,7 @@ public byte[] encode(Object obj) { } } - /** + /* * Decode. */ @Override @@ -1158,7 +1172,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return decodeText(buf, type); } - /** + /* * Get encoded size. */ @Override @@ -1166,7 +1180,7 @@ public int getEncodedSize(Object obj) { return getTextEncodedSize(obj); } - /** + /* * Get default obj meta. */ @Override @@ -1175,7 +1189,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_UTF8MB4_GENERAL_CI, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1186,10 +1200,18 @@ public Comparable parseToComparable(Object o, ObCollationType ct) return parseTextToComparable(this, o, ct); } + /* + * Parse to bytes + */ + @Override + public byte[] parseToBytes(Object o, ObCollationType ct) throws IllegalArgumentException, FeatureNotSupportedException { + return parseTextToBytes(this, o, ct); + } + }, // charset: utf8mb4 or binary ObCharType(23) { // TODO not support - /** + /* * Encode. */ @Override @@ -1197,7 +1219,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVString((String) obj); } - /** + /* * Decode. */ @Override @@ -1205,7 +1227,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return decodeText(buf, type); } - /** + /* * Get encoded size. */ @Override @@ -1213,7 +1235,7 @@ public int getEncodedSize(Object obj) { return getTextEncodedSize(obj); } - /** + /* * Get default obj meta. */ @Override @@ -1222,7 +1244,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_UTF8MB4_GENERAL_CI, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1232,11 +1254,19 @@ public Comparable parseToComparable(Object o, ObCollationType ct) return parseTextToComparable(this, o, ct); } + /* + * Parse to bytes + */ + @Override + public byte[] parseToBytes(Object o, ObCollationType ct) throws IllegalArgumentException, FeatureNotSupportedException { + return parseTextToBytes(this, o, ct); + } + }, // charset: utf8mb4 or binary @Deprecated ObHexStringType(24) { - /** + /* * Encode. */ @Override @@ -1244,7 +1274,7 @@ public byte[] encode(Object obj) { return new byte[0]; } - /** + /* * Decode. */ @Override @@ -1252,7 +1282,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return null; } - /** + /* * Get encoded size. */ @Override @@ -1260,7 +1290,7 @@ public int getEncodedSize(Object obj) { return 0; } - /** + /* * Get default obj meta. */ @Override @@ -1269,7 +1299,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1283,7 +1313,7 @@ public String parseToComparable(Object o, ObCollationType ct) ObExtendType(25) { // MIN_OBJECT / MAX_OBJECT - /** + /* * Encode. */ @Override @@ -1291,7 +1321,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi64((Long) obj); } - /** + /* * Decode. */ @Override @@ -1299,7 +1329,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeVi64(buf); } - /** + /* * Get encoded size. */ @Override @@ -1307,7 +1337,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes((Long) obj); } - /** + /* * Get default obj meta. */ @Override @@ -1316,7 +1346,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1329,7 +1359,7 @@ public Comparable parseToComparable(Object o, ObCollationType ct) }, // Min, Max, NOP etc. @Deprecated ObUnknownType(26) { - /** + /* * Encode. */ @Override @@ -1337,7 +1367,7 @@ public byte[] encode(Object obj) { return new byte[0]; } - /** + /* * Decode. */ @Override @@ -1345,7 +1375,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return null; } - /** + /* * Get encoded size. */ @Override @@ -1353,7 +1383,7 @@ public int getEncodedSize(Object obj) { return 0; } - /** + /* * Get default obj meta. */ @Override @@ -1362,7 +1392,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1375,7 +1405,7 @@ public Comparable parseToComparable(Object o, ObCollationType ct) }, // For question mark(?) in prepared statement, no need to serialize // @note future new types to be defined here !!! ObTinyTextType(27) { - /** + /* * Encode. */ @Override @@ -1383,7 +1413,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVString((String) obj); } - /** + /* * Decode. */ @Override @@ -1391,7 +1421,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return decodeText(buf, type); } - /** + /* * Get encoded size. */ @Override @@ -1399,7 +1429,7 @@ public int getEncodedSize(Object obj) { return getTextEncodedSize(obj); } - /** + /* * Get default obj meta. */ @Override @@ -1408,7 +1438,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1419,9 +1449,17 @@ public Comparable parseToComparable(Object o, ObCollationType ct) return parseTextToComparable(this, o, ct); } + /* + * Parse to bytes + */ + @Override + public byte[] parseToBytes(Object o, ObCollationType ct) throws IllegalArgumentException, FeatureNotSupportedException { + return parseTextToBytes(this, o, ct); + } + }, ObTextType(28) { - /** + /* * Encode. */ @Override @@ -1430,7 +1468,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVString((String) obj); } - /** + /* * Decode. */ @Override @@ -1438,7 +1476,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return decodeText(buf, type); } - /** + /* * Get encoded size. */ @Override @@ -1446,7 +1484,7 @@ public int getEncodedSize(Object obj) { return getTextEncodedSize(obj); } - /** + /* * Get default obj meta. */ @Override @@ -1455,7 +1493,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1465,9 +1503,17 @@ public Comparable parseToComparable(Object o, ObCollationType ct) return parseTextToComparable(this, o, ct); } + /* + * Parse to bytes + */ + @Override + public byte[] parseToBytes(Object o, ObCollationType ct) throws IllegalArgumentException, FeatureNotSupportedException { + return parseTextToBytes(this, o, ct); + } + }, ObMediumTextType(29) { - /** + /* * Encode. */ @Override @@ -1476,7 +1522,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVString((String) obj); } - /** + /* * Decode. */ @Override @@ -1484,7 +1530,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return decodeText(buf, type); } - /** + /* * Get encoded size. */ @Override @@ -1492,7 +1538,7 @@ public int getEncodedSize(Object obj) { return getTextEncodedSize(obj); } - /** + /* * Get default obj meta. */ @Override @@ -1501,7 +1547,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1509,9 +1555,17 @@ public Comparable parseToComparable(Object o, ObCollationType ct) { return parseTextToComparable(this, o, ct); } + /* + * Parse to bytes + */ + @Override + public byte[] parseToBytes(Object o, ObCollationType ct) throws IllegalArgumentException, FeatureNotSupportedException { + return parseTextToBytes(this, o, ct); + } + }, ObLongTextType(30) { - /** + /* * Encode. */ @Override @@ -1520,7 +1574,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVString((String) obj); } - /** + /* * Decode. */ @Override @@ -1528,7 +1582,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return decodeText(buf, type); } - /** + /* * Get encoded size. */ @Override @@ -1536,7 +1590,7 @@ public int getEncodedSize(Object obj) { return getTextEncodedSize(obj); } - /** + /* * Get default obj meta. */ @Override @@ -1545,17 +1599,26 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override public Comparable parseToComparable(Object o, ObCollationType ct) { return parseTextToComparable(this, o, ct); } + + /* + * Parse to bytes + */ + @Override + public byte[] parseToBytes(Object o, ObCollationType ct) throws IllegalArgumentException, FeatureNotSupportedException { + return parseTextToBytes(this, o, ct); + } + }, ObBitType(31) { // TODO not support - /** + /* * Encode. */ @Override @@ -1564,7 +1627,7 @@ public byte[] encode(Object obj) { return Serialization.encodeVi64(((Number) obj).longValue()); } - /** + /* * Decode. */ @Override @@ -1573,7 +1636,7 @@ public Object decode(ByteBuf buf, ObCollationType type) { return Serialization.decodeVi64(buf); } - /** + /* * Get encoded size. */ @Override @@ -1582,7 +1645,7 @@ public int getEncodedSize(Object obj) { return Serialization.getNeedBytes(((Number) obj).longValue()); } - /** + /* * Get default obj meta. */ @Override @@ -1592,7 +1655,7 @@ public ObObjMeta getDefaultObjMeta() { ObCollationType.CS_TYPE_BINARY, (byte) 10); } - /** + /* * Parse to comparable. */ @Override @@ -1618,14 +1681,14 @@ public Comparable parseToComparable(Object o, ObCollationType ct) { } } - /** + /* * Default obj meta. */ public static ObObjMeta defaultObjMeta(Object object) { return valueOfType(object).getDefaultObjMeta(); } - /** + /* * Value of type. */ public static ObObjType valueOfType(Object object) { @@ -1665,21 +1728,21 @@ public static ObObjType valueOfType(Object object) { + object.getClass().getName()); } - /** + /* * Value of. */ public static ObObjType valueOf(int value) { return map.get(value); } - /** + /* * Get value. */ public int getValue() { return value; } - /** + /* * Get byte value. */ public byte getByteValue() { @@ -1697,8 +1760,13 @@ public byte getByteValue() { public abstract Comparable parseToComparable(Object o, ObCollationType ct) throws IllegalArgumentException, FeatureNotSupportedException; + public byte[] parseToBytes(Object o, ObCollationType ct) throws IllegalArgumentException, FeatureNotSupportedException { + // do nothing now + // We have implemented some parseToBytes() for varchar and varbinary type + return null; + } - /** + /* * Parse to long or null. */ public static Long parseToLongOrNull(Object object) { @@ -1721,7 +1789,7 @@ public static Long parseToLongOrNull(Object object) { return value; } - /** + /* * Decode text. */ public Object decodeText(ByteBuf buf, ObCollationType type) { @@ -1732,7 +1800,7 @@ public Object decodeText(ByteBuf buf, ObCollationType type) { } } - /** + /* * Get text encoded size. */ public static int getTextEncodedSize(Object obj) { @@ -1746,7 +1814,57 @@ public static int getTextEncodedSize(Object obj) { } } - /** + /* + * Parse text to comparable. + */ + public static byte[] parseTextToBytes(ObObjType obObjType, Object object, + ObCollationType collationType) { + if (collationType == ObCollationType.CS_TYPE_BINARY) { + if (object instanceof ObBytesString) { + return ((ObBytesString) object).bytes; + } + + if (object instanceof byte[]) { + return (new ObBytesString((byte[]) object)).bytes; + } + + if (object instanceof String) { + return (new ObBytesString((String) object)).bytes; + } + + if (object instanceof ObVString) { + return (new ObBytesString(((ObVString) object).getBytesVal())).bytes; + } + } else { + + if (object instanceof String) { + return ((String) object).getBytes(); + } + if (object instanceof ObBytesString) { + return (Serialization.decodeVString(((ObBytesString) object).bytes)).getBytes(); + } + + if (object instanceof byte[]) { + try { + return (Serialization.decodeVString((byte[]) object)).getBytes(); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + obObjType.name() + + "can not parseToComparable byte array to string with utf-8 charset", + e); + } + } + + if (object instanceof ObVString) { + return (((ObVString) object).getStringVal()).getBytes(); + } + } + + throw new IllegalArgumentException(obObjType.name() + "can not parseToComparable argument:" + + object); + } + + /* * Parse text to comparable. */ public static Comparable parseTextToComparable(ObObjType obObjType, Object object, @@ -1783,7 +1901,8 @@ public static Comparable parseTextToComparable(ObObjType obObjType, Object objec } catch (IllegalArgumentException e) { throw new IllegalArgumentException( obObjType.name() - + "can not parseToComparable byte array to string with utf-8 charset"); + + "can not parseToComparable byte array to string with utf-8 charset", + e); } } @@ -1796,7 +1915,7 @@ public static Comparable parseTextToComparable(ObObjType obObjType, Object objec + object); } - /** + /* * Parse timestamp. */ public static Timestamp parseTimestamp(ObObjType obObjType, Object object, @@ -1817,11 +1936,11 @@ public static Timestamp parseTimestamp(ObObjType obObjType, Object object, return new Timestamp((Long) object); } - throw new IllegalArgumentException(obObjType.name() + "can not parseToComparable with " + throw new IllegalArgumentException(obObjType.name() + " can not parseToComparable with " + collationType + "argument:" + object); } - /** + /* * Parse long. */ public static Long parseLong(ObObjType obObjType, Object object, ObCollationType collationType) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObRowKey.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObRowKey.java index 78e8fdeb..619f5101 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObRowKey.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObRowKey.java @@ -31,42 +31,46 @@ public class ObRowKey { private List objs = new ArrayList(); - /** + /* * Get obj count. */ public long getObjCount() { return objs.size(); } - /** + /* * Get objs. */ public List getObjs() { return objs; } - /** + public void setObjs(List objs) { + this.objs = objs; + } + + /* * Get obj. */ public ObObj getObj(int idx) { return objs.get(idx); } - /** + /* * Set obj. */ public void setObj(int idx, ObObj obObj) { objs.set(idx, obObj); } - /** + /* * Add obj. */ public void addObj(ObObj obObj) { objs.add(obObj); } - /** + /* * Get instance. */ public static ObRowKey getInstance(ObObj... values) { @@ -78,7 +82,7 @@ public static ObRowKey getInstance(ObObj... values) { return rowKey; } - /** + /* * Get instance. */ public static ObRowKey getInstance(Object... values) { @@ -94,7 +98,7 @@ public static ObRowKey getInstance(Object... values) { return rowKey; } - /** + /* * Get instance. */ public static ObRowKey getInstance(List values) { @@ -110,7 +114,7 @@ public static ObRowKey getInstance(List values) { return rowKey; } - /** + /* * Equals. */ @Override @@ -123,11 +127,12 @@ public boolean equals(Object o) { return Objects.equal(objs, obRowKey.objs); } - /** + /* * Hash code. */ @Override public int hashCode() { return Objects.hashCode(objs); } + } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java new file mode 100644 index 00000000..0a00e69b --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableObjType.java @@ -0,0 +1,360 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl; + +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import java.util.HashMap; +import java.util.Map; + +public enum ObTableObjType { + + ObTableNullType(0) { + }, + + ObTableTinyIntType(1) { + }, + + ObTableSmallIntType(2) { + }, + + ObTableInt32Type(3) { + }, + + ObTableInt64Type(4) { + }, + + ObTableVarcharType(5) { + }, + + ObTableVarbinaryType(6) { + @Override + public void decode(ByteBuf buf, ObObj obj) { + ObObjType objType = getObjType(this); + ObObjMeta objMeta = objType.getDefaultObjMeta(); + objMeta.setCsType(ObCollationType.CS_TYPE_BINARY); + obj.setMeta(objMeta); + obj.setValueFromTableObj(objType.decode(buf, objMeta.getCsType())); + } + }, + + ObTableDoubleType(7) { + }, + + ObTableFloatType(8) { + }, + + ObTableTimestampType(9) { + @Override + public byte[] encode(ObObj obj) { + return encodeWithMeta(obj); + } + + @Override + public void decode(ByteBuf buf, ObObj obj) { + decodeWithMeta(buf, obj); + } + + @Override + public int getEncodedSize(ObObj obj) { + return getEncodedSizeWithMeta(obj); + } + }, + + ObTableDateTimeType(10) { + @Override + public byte[] encode(ObObj obj) { + return encodeWithMeta(obj); + } + + @Override + public void decode(ByteBuf buf, ObObj obj) { + decodeWithMeta(buf, obj); + } + + @Override + public int getEncodedSize(ObObj obj) { + return getEncodedSizeWithMeta(obj); + } + }, + + ObTableMinType(11) { + public byte[] encode(ObObj obj) { + byte[] bytes = new byte[this.getEncodedSize(obj)]; + int idx = 0; + System.arraycopy(Serialization.encodeI8(this.getValue()), 0, bytes, idx, 1); + idx += 1; + return bytes; + } + + public void decode(ByteBuf buf, ObObj obj) { + ObObjType objType = getObjType(this); + ObObjMeta objMeta = objType.getDefaultObjMeta(); + obj.setMeta(objMeta); + obj.setValueFromTableObj(-2L); + } + + public int getEncodedSize(ObObj obj) { + return DEFAULT_TABLE_OBJ_TYPE_SIZE; + } + }, + + ObTableMaxType(12) { + public byte[] encode(ObObj obj) { + byte[] bytes = new byte[this.getEncodedSize(obj)]; + int idx = 0; + System.arraycopy(Serialization.encodeI8(this.getValue()), 0, bytes, idx, 1); + idx += 1; + return bytes; + } + + public void decode(ByteBuf buf, ObObj obj) { + ObObjType objType = getObjType(this); + ObObjMeta objMeta = objType.getDefaultObjMeta(); + obj.setMeta(objMeta); + obj.setValueFromTableObj(-3L); + } + + public int getEncodedSize(ObObj obj) { + return DEFAULT_TABLE_OBJ_TYPE_SIZE; + } + }, + + // 13 ObTableUTinyIntType + // 14 ObTableUSmallIntType + // 15 ObTableUInt32Type + // 16 ObTableUInt64Type + + ObTableTinyTextType(17) { + public void decode(ByteBuf buf, ObObj obj) { + decodeWithUtf8(buf, obj); + } + }, + + ObTableTextType(18) { + public void decode(ByteBuf buf, ObObj obj) { + decodeWithUtf8(buf, obj); + } + }, + + ObTableMediumTextType(19) { + public void decode(ByteBuf buf, ObObj obj) { + decodeWithUtf8(buf, obj); + } + }, + + ObTableLongTextType(20) { + public void decode(ByteBuf buf, ObObj obj) { + decodeWithUtf8(buf, obj); + } + }, + + ObTableTinyBlobType(21) { + }, + + ObTableBlobType(22) { + }, + + ObTableMediumBlobType(23) { + }, + + ObTableLongBlobType(24) { + }, + + ObTableInvalidType(25) { + }; + + private int value; + // mapping from value to enum + private static Map valueMap = new HashMap(); + // mapping from ObTableObjType to ObObjType + private static Map tableObjTypeMap = new HashMap<>(); + + ObTableObjType(int value) { + this.value = value; + } + + static { + for (ObTableObjType type : ObTableObjType.values()) { + valueMap.put(type.value, type); + } + } + + public static ObTableObjType getTableObjType(ObObj obj) { + ObObjType objType = obj.getMeta().getType(); + ObCollationType objCsType = obj.getMeta().getCsType(); + if (objType == ObObjType.ObNullType) { + // only for GET operation default value + return ObTableNullType; + } else if (objType == ObObjType.ObTinyIntType) { + return ObTableTinyIntType; + } else if (objType == ObObjType.ObSmallIntType) { + return ObTableObjType.ObTableSmallIntType; + } else if (objType == ObObjType.ObInt32Type) { + return ObTableObjType.ObTableInt32Type; + } else if (objType == ObObjType.ObInt64Type) { + return ObTableObjType.ObTableInt64Type; + } else if (objType == ObObjType.ObVarcharType) { + if (objCsType == ObCollationType.CS_TYPE_BINARY) { + return ObTableObjType.ObTableVarbinaryType; + } else { + return ObTableObjType.ObTableVarcharType; + } + } else if (objType == ObObjType.ObDoubleType) { + return ObTableObjType.ObTableDoubleType; + } else if (objType == ObObjType.ObFloatType) { + return ObTableObjType.ObTableFloatType; + } else if (objType == ObObjType.ObTimestampType) { + return ObTableObjType.ObTableTimestampType; + } else if (objType == ObObjType.ObDateTimeType) { + return ObTableObjType.ObTableDateTimeType; + } else if (objType == ObObjType.ObExtendType) { + if (obj.isMinObj()) { + return ObTableObjType.ObTableMinType; + } else if (obj.isMaxObj()) { + return ObTableObjType.ObTableMaxType; + } + } + + throw new IllegalArgumentException("cannot get ObTableObjType, invalid ob obj type: " + + objType.getClass().getName()); + } + + static { + tableObjTypeMap.put(ObTableNullType, ObObjType.ObNullType); + tableObjTypeMap.put(ObTableTinyIntType, ObObjType.ObTinyIntType); + tableObjTypeMap.put(ObTableSmallIntType, ObObjType.ObSmallIntType); + tableObjTypeMap.put(ObTableInt32Type, ObObjType.ObInt32Type); + tableObjTypeMap.put(ObTableInt64Type, ObObjType.ObInt64Type); + tableObjTypeMap.put(ObTableVarcharType, ObObjType.ObVarcharType); + tableObjTypeMap.put(ObTableVarbinaryType, ObObjType.ObVarcharType); + tableObjTypeMap.put(ObTableDoubleType, ObObjType.ObDoubleType); + tableObjTypeMap.put(ObTableFloatType, ObObjType.ObFloatType); + tableObjTypeMap.put(ObTableTimestampType, ObObjType.ObTimestampType); + tableObjTypeMap.put(ObTableDateTimeType, ObObjType.ObDateTimeType); + tableObjTypeMap.put(ObTableTinyTextType, ObObjType.ObTinyTextType); + tableObjTypeMap.put(ObTableTextType, ObObjType.ObTextType); + tableObjTypeMap.put(ObTableMediumTextType, ObObjType.ObMediumTextType); + tableObjTypeMap.put(ObTableLongTextType, ObObjType.ObLongTextType); + tableObjTypeMap.put(ObTableTinyBlobType, ObObjType.ObTinyTextType); + tableObjTypeMap.put(ObTableBlobType, ObObjType.ObTextType); + tableObjTypeMap.put(ObTableMediumBlobType, ObObjType.ObMediumTextType); + tableObjTypeMap.put(ObTableLongBlobType, ObObjType.ObLongTextType); + tableObjTypeMap.put(ObTableMinType, ObObjType.ObExtendType); + tableObjTypeMap.put(ObTableMaxType, ObObjType.ObExtendType); + } + + public static ObObjType getObjType(ObTableObjType tableObjType) { + ObObjType objType = tableObjTypeMap.get(tableObjType); + if (objType == null) { + throw new IllegalArgumentException("cannot get ObTableObjType, invalid table obj type: " + + tableObjType.getClass().getName()); + } + return objType; + } + + /* + * Value of. + */ + public static ObTableObjType valueOf(int value) { + return valueMap.get(value); + } + + /* + * Get value. + */ + public byte getValue() { + return (byte) value; + } + + public byte[] encode(ObObj obj) { + ObObjType objType = obj.getMeta().getType(); + byte[] bytes = new byte[this.getEncodedSize(obj)]; + int idx = 0; + System.arraycopy(Serialization.encodeI8(this.getValue()), 0, bytes, idx, 1); + idx += 1; + + byte[] valueBytes = objType.encode(obj.getValue()); + System.arraycopy(valueBytes, 0, bytes, idx, valueBytes.length); + idx += valueBytes.length; + + return bytes; + } + + public void decode(ByteBuf buf, ObObj obj) { + ObObjType objType = getObjType(this); + ObObjMeta objMeta = objType.getDefaultObjMeta(); + obj.setMeta(objMeta); + obj.setValueFromTableObj(objType.decode(buf, objMeta.getCsType())); + } + + public int getEncodedSize(ObObj obj) { + ObObjType objType = obj.getMeta().getType(); + return DEFAULT_TABLE_OBJ_TYPE_SIZE + objType.getEncodedSize(obj.getValue()); + } + + public byte[] encodeWithMeta(ObObj obj) { + ObObjType objType = obj.getMeta().getType(); + ObTableObjType tableObjType = getTableObjType(obj); + byte[] bytes = new byte[tableObjType.getEncodedSize(obj)]; + int idx = 0; + System.arraycopy(Serialization.encodeI8(tableObjType.getValue()), 0, bytes, idx, 1); + idx += 1; + System.arraycopy(Serialization.encodeI8(obj.getMeta().getCsLevel().getByteValue()), 0, + bytes, idx, 1); + idx += 1; + System.arraycopy(Serialization.encodeI8(obj.getMeta().getCsType().getByteValue()), 0, + bytes, idx, 1); + idx += 1; + System.arraycopy(Serialization.encodeI8(obj.getMeta().getScale()), 0, bytes, idx, 1); + idx += 1; + byte[] valueBytes = objType.encode(obj.getValue()); + System.arraycopy(valueBytes, 0, bytes, idx, valueBytes.length); + idx += valueBytes.length; + + return bytes; + } + + public void decodeWithMeta(ByteBuf buf, ObObj obj) { + ObObjType objType = getObjType(this); + ObObjMeta meta = obj.getMeta(); + meta.setType(objType); + meta.setCsLevel(ObCollationLevel.valueOf(Serialization.decodeI8(buf.readByte()))); + meta.setCsType(ObCollationType.valueOf(Serialization.decodeI8(buf.readByte()))); + meta.setScale(Serialization.decodeI8(buf.readByte())); + obj.setValueFromTableObj(objType.decode(buf, meta.getCsType())); + } + + public int getEncodedSizeWithMeta(ObObj obj) { + ObObjType objType = getObjType(this); + int len = DEFAULT_TABLE_OBJ_META_SIZE + objType.getEncodedSize(obj.getValue()); + return len; + } + + public void decodeWithUtf8(ByteBuf buf, ObObj obj) { + ObObjType objType = getObjType(this); + ObObjMeta objMeta = objType.getDefaultObjMeta(); + objMeta.setCsType(ObCollationType.CS_TYPE_UTF8MB4_GENERAL_CI); + obj.setMeta(objMeta); + obj.setValueFromTableObj(objType.decode(buf, objMeta.getCsType())); + } + + public static int DEFAULT_TABLE_OBJ_TYPE_SIZE = 1; + public static int DEFAULT_TABLE_OBJ_META_SIZE = 4; +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableSerialUtil.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableSerialUtil.java new file mode 100644 index 00000000..e32c5ffb --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/ObTableSerialUtil.java @@ -0,0 +1,153 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl; + +import com.alipay.oceanbase.rpc.ObGlobal; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObBorderFlag; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import static com.alipay.oceanbase.rpc.protocol.payload.impl.ObTableObjType.*; + +public class ObTableSerialUtil { + static public int getEncodedSize(ObObj obj) { + return getTableObjType(obj).getEncodedSize(obj); + } + + static public byte[] encode(ObObj obj) { + return getTableObjType(obj).encode(obj); + } + + static public void decode(ByteBuf buf, ObObj obj) { + ObTableObjType tableObjType = decodeTableObjType(buf); + tableObjType.decode(buf, obj); + } + + public static ObTableObjType decodeTableObjType(ByteBuf buf) { + if (buf == null) { + throw new IllegalArgumentException("cannot get ObTableObjType, buf is null"); + } + byte type = Serialization.decodeI8(buf); + ObTableObjType objType = ObTableObjType.valueOf(type); + if (objType == null) { + throw new IllegalArgumentException("cannot get table object type from value"); + } + return objType; + } + + static public int getEncodedSize(ObNewRange range) { + int encodedSize = 0; + encodedSize += Serialization.getNeedBytes(range.getTableId()); + encodedSize += 1; // borderFlag + ObRowKey startKey = range.getStartKey(); + long rowkeySize = startKey.getObjCount(); + encodedSize += Serialization.getNeedBytes(rowkeySize); + for (int i = 0; i < rowkeySize; i++) { + ObObj obObj = startKey.getObj(i); + encodedSize += ObTableSerialUtil.getEncodedSize(obObj); + } + + ObRowKey endKey = range.getEndKey(); + rowkeySize = endKey.getObjCount(); + encodedSize += Serialization.getNeedBytes(rowkeySize); + for (int i = 0; i < rowkeySize; i++) { + ObObj obObj = endKey.getObj(i); + encodedSize += ObTableSerialUtil.getEncodedSize(obObj); + } + + if (ObGlobal.obVsnMajor() >= 4) { + encodedSize += Serialization.getNeedBytes(range.getFlag()); + } + + return encodedSize; + } + + static public byte[] encode(ObNewRange range) { + byte[] bytes = new byte[getEncodedSize(range)]; + int idx = 0; + + long tableId = range.getTableId(); + int len = Serialization.getNeedBytes(tableId); + System.arraycopy(Serialization.encodeVi64(tableId), 0, bytes, idx, len); + idx += len; + System + .arraycopy(Serialization.encodeI8(range.getBorderFlag().getValue()), 0, bytes, idx, 1); + idx += 1; + + ObRowKey startKey = range.getStartKey(); + long rowkeySize = startKey.getObjCount(); + len = Serialization.getNeedBytes(rowkeySize); + System.arraycopy(Serialization.encodeVi64(rowkeySize), 0, bytes, idx, len); + idx += len; + for (int i = 0; i < rowkeySize; i++) { + ObObj obObj = startKey.getObj(i); + byte[] objBytes = ObTableSerialUtil.encode(obObj); + System.arraycopy(objBytes, 0, bytes, idx, objBytes.length); + idx += objBytes.length; + } + + ObRowKey endKey = range.getEndKey(); + rowkeySize = endKey.getObjCount(); + len = Serialization.getNeedBytes(rowkeySize); + System.arraycopy(Serialization.encodeVi64(rowkeySize), 0, bytes, idx, len); + idx += len; + for (int i = 0; i < rowkeySize; i++) { + ObObj obObj = endKey.getObj(i); + byte[] objBytes = ObTableSerialUtil.encode(obObj); + System.arraycopy(objBytes, 0, bytes, idx, objBytes.length); + idx += objBytes.length; + } + + if (ObGlobal.obVsnMajor() >= 4) { + long flag = range.getFlag(); + len = Serialization.getNeedBytes(flag); + System.arraycopy(Serialization.encodeVi64(flag), 0, bytes, idx, len); + idx += len; + } + + return bytes; + } + + static public void decode(ByteBuf buf, ObNewRange range) { + range.setTableId(Serialization.decodeVi64(buf)); + range.setBorderFlag(ObBorderFlag.valueOf(Serialization.decodeI8(buf.readByte()))); + + long rowkeySize = Serialization.decodeVi64(buf); + ObRowKey startKey = new ObRowKey(); + for (int i = 0; i < rowkeySize; i++) { + ObObj obObj = new ObObj(); + decode(buf, obObj); + startKey.addObj(obObj); + } + range.setStartKey(startKey); + + rowkeySize = Serialization.decodeVi64(buf); + ObRowKey endKey = new ObRowKey(); + for (int i = 0; i < rowkeySize; i++) { + ObObj obObj = new ObObj(); + decode(buf, obObj); + endKey.addObj(obObj); + } + range.setEndKey(endKey); + + if (ObGlobal.obVsnMajor() >= 4) { + range.setFlag(Serialization.decodeVi64(buf)); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumn.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumn.java index a69c799d..3241c109 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumn.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumn.java @@ -25,7 +25,7 @@ public class ObGeneratedColumn extends ObColumn { - /** + /* * Ob generated column. */ public ObGeneratedColumn(String columnName, int index, ObObjType obObjType, @@ -35,7 +35,7 @@ public ObGeneratedColumn(String columnName, int index, ObObjType obObjType, columnExpress); } - /** + /* * Eval value. */ public Object evalValue(Object... refs) throws IllegalArgumentException { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumnReferFunc.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumnReferFunc.java index 2163002e..4530d654 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumnReferFunc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumnReferFunc.java @@ -25,7 +25,7 @@ public class ObGeneratedColumnReferFunc implements ObGeneratedColumnSimpleFunc { private final List refColumnNames; - /** + /* * Ob generated column refer func. */ public ObGeneratedColumnReferFunc(String columnName) { @@ -34,7 +34,7 @@ public ObGeneratedColumnReferFunc(String columnName) { this.refColumnNames = refColumnNameList; } - /** + /* * Set parameters. */ @Override @@ -42,7 +42,7 @@ public void setParameters(List parameters) { //ignore } - /** + /* * Get min parameters. */ @Override @@ -50,7 +50,7 @@ public int getMinParameters() { return 0; } - /** + /* * Get max parameters. */ @Override @@ -58,7 +58,7 @@ public int getMaxParameters() { return 0; } - /** + /* * Get ref column names. */ @Override @@ -66,7 +66,7 @@ public List getRefColumnNames() { return refColumnNames; } - /** + /* * Eval value. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumnSubStrFunc.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumnSubStrFunc.java index 15b08b1b..5854ceb1 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumnSubStrFunc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObGeneratedColumnSubStrFunc.java @@ -32,35 +32,35 @@ public class ObGeneratedColumnSubStrFunc implements ObGeneratedColumnSimpleFunc private int pos = 0; private int len = Integer.MIN_VALUE; - /** + /* * Get pos. */ public int getPos() { return pos; } - /** + /* * Set pos. */ public void setPos(int pos) { this.pos = pos; } - /** + /* * Get len. */ public int getLen() { return len; } - /** + /* * Set len. */ public void setLen(int len) { this.len = len; } - /** + /* * Set parameters. */ @Override @@ -108,7 +108,7 @@ public void setParameters(List parameters) throws IllegalArgumentExcepti } } - /** + /* * Get min parameters. */ @Override @@ -116,7 +116,7 @@ public int getMinParameters() { return 2; } - /** + /* * Get max parameters. */ @Override @@ -124,7 +124,7 @@ public int getMaxParameters() { return 3; } - /** + /* * Get ref column names. */ @Override @@ -132,7 +132,7 @@ public List getRefColumnNames() { return refColumnNames; } - /** + /* * Eval value. */ @Override @@ -232,7 +232,7 @@ public Object evalValue(ObCollationType collationType, Object... refs) } } - /** + /* * To string. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObSimpleColumn.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObSimpleColumn.java index 1a646013..28642de8 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObSimpleColumn.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/column/ObSimpleColumn.java @@ -40,7 +40,7 @@ public class ObSimpleColumn extends ObColumn { ObObjType.ObVarcharType, ObCollationType.CS_TYPE_BINARY); - /** + /* * Ob simple column. */ public ObSimpleColumn(String columnName, int index, ObObjType obObjType, @@ -49,7 +49,7 @@ public ObSimpleColumn(String columnName, int index, ObObjType obObjType, null); } - /** + /* * Eval value. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObLoadDupActionType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObLoadDupActionType.java new file mode 100644 index 00000000..359948b0 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObLoadDupActionType.java @@ -0,0 +1,52 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import java.util.HashMap; +import java.util.Map; + +public enum ObLoadDupActionType { + + STOP_ON_DUP(0), REPLACE(1), IGNORE(2), INVALID_MODE(3); + + private final int value; + private static final Map map = new HashMap(); + + static { + for (ObLoadDupActionType type : ObLoadDupActionType.values()) { + map.put(type.value, type); + } + } + + public static ObLoadDupActionType valueOf(int value) { + return map.get(value); + } + + ObLoadDupActionType(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + + public byte getByteValue() { + return (byte) value; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadAbortArg.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadAbortArg.java new file mode 100644 index 00000000..26ccd0a8 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadAbortArg.java @@ -0,0 +1,92 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadAbortArg, +// table_id_, +// task_id_); + +public class ObTableDirectLoadAbortArg implements ObSimplePayload { + + private long tableId = 0; + private long taskId = 0; + + public ObTableDirectLoadAbortArg() { + } + + public long getTableId() { + return tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + } + + public long getTaskId() { + return taskId; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVi64(buf, tableId); + Serialization.encodeVi64(buf, taskId); + } + + /** + * Decode. + */ + @Override + public ObTableDirectLoadAbortArg decode(ByteBuf buf) { + tableId = Serialization.decodeVi64(buf); + taskId = Serialization.decodeVi64(buf); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + return Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(taskId); + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadBeginArg.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadBeginArg.java new file mode 100644 index 00000000..945f253e --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadBeginArg.java @@ -0,0 +1,160 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadBeginArg, +// table_name_, +// parallel_, +// max_error_row_count_, +// dup_action_, +// timeout_, +// heartbeat_timeout_, +// force_create_); + +public class ObTableDirectLoadBeginArg implements ObSimplePayload { + + private String tableName; + private long parallel = 0; + private long maxErrorRowCount = 0; + private ObLoadDupActionType dupAction = ObLoadDupActionType.INVALID_MODE; + private long timeout = 0; + private long heartBeatTimeout = 0; + private boolean forceCreate = false; + + public ObTableDirectLoadBeginArg() { + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public long getParallel() { + return parallel; + } + + public void setParallel(long parallel) { + this.parallel = parallel; + } + + public long getMaxErrorRowCount() { + return maxErrorRowCount; + } + + public void setMaxErrorRowCount(long maxErrorRowCount) { + this.maxErrorRowCount = maxErrorRowCount; + } + + public ObLoadDupActionType getDupAction() { + return dupAction; + } + + public void setDupAction(ObLoadDupActionType dupAction) { + this.dupAction = dupAction; + } + + public long getTimeout() { + return timeout; + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + public long getHeartBeatTimeout() { + return heartBeatTimeout; + } + + public void setHeartBeatTimeout(long heartBeatTimeout) { + this.heartBeatTimeout = heartBeatTimeout; + } + + public boolean getForceCreate() { + return forceCreate; + } + + public void setForceCreate(boolean forceCreate) { + this.forceCreate = forceCreate; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVString(buf, tableName); + Serialization.encodeVi64(buf, parallel); + Serialization.encodeVi64(buf, maxErrorRowCount); + Serialization.encodeI8(buf, dupAction.getByteValue()); + Serialization.encodeVi64(buf, timeout); + Serialization.encodeVi64(buf, heartBeatTimeout); + Serialization.encodeI8(buf, (byte) (forceCreate ? 1 : 0)); + } + + /** + * Decode. + */ + @Override + public ObTableDirectLoadBeginArg decode(ByteBuf buf) { + tableName = Serialization.decodeVString(buf); + parallel = Serialization.decodeVi64(buf); + maxErrorRowCount = Serialization.decodeVi64(buf); + dupAction = ObLoadDupActionType.valueOf(Serialization.decodeI8(buf)); + timeout = Serialization.decodeVi64(buf); + heartBeatTimeout = Serialization.decodeVi64(buf); + forceCreate = (Serialization.decodeI8(buf) != 0); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + int len = 0; + len += Serialization.getNeedBytes(tableName); + len += Serialization.getNeedBytes(parallel); + len += Serialization.getNeedBytes(maxErrorRowCount); + len += 1; /*dupAction*/ + len += Serialization.getNeedBytes(timeout); + len += Serialization.getNeedBytes(heartBeatTimeout); + len += 1; /*forceCreate*/ + return len; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadBeginRes.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadBeginRes.java new file mode 100644 index 00000000..a4da51c2 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadBeginRes.java @@ -0,0 +1,148 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadBeginRes, +// table_id_, +// task_id_, +// column_names_, +// status_, +// error_code_); + +public class ObTableDirectLoadBeginRes implements ObSimplePayload { + + private long tableId = 0; + private long taskId = 0; + private String[] columnNames = new String[0]; + private ObTableLoadClientStatus status = ObTableLoadClientStatus.MAX_STATUS; + private int errorCode = ResultCodes.OB_SUCCESS.errorCode; + + public ObTableDirectLoadBeginRes() { + } + + public long getTableId() { + return tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + } + + public long getTaskId() { + return taskId; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + public String[] getColumnNames() { + return columnNames; + } + + public void setColumnNames(String[] columnNames) { + if (columnNames == null) { + throw new NullPointerException(); + } + this.columnNames = columnNames; + } + + public ObTableLoadClientStatus getStatus() { + return status; + } + + public void setStatus(ObTableLoadClientStatus status) { + this.status = status; + } + + public int getErrorCode() { + return errorCode; + } + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVi64(buf, tableId); + Serialization.encodeVi64(buf, taskId); + Serialization.encodeVi32(buf, columnNames.length); + for (int i = 0; i < columnNames.length; ++i) { + Serialization.encodeVString(columnNames[i]); + } + Serialization.encodeI8(buf, status.getByteValue()); + Serialization.encodeVi32(buf, errorCode); + } + + /** + * Decode. + */ + @Override + public ObTableDirectLoadBeginRes decode(ByteBuf buf) { + tableId = Serialization.decodeVi64(buf); + taskId = Serialization.decodeVi64(buf); + int columnNameCount = Serialization.decodeVi32(buf); + columnNames = new String[columnNameCount]; + for (int i = 0; i < columnNameCount; ++i) { + columnNames[i] = Serialization.decodeVString(buf); + } + status = ObTableLoadClientStatus.valueOf(Serialization.decodeI8(buf)); + errorCode = Serialization.decodeVi32(buf); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + int len = 0; + len += Serialization.getNeedBytes(tableId); + len += Serialization.getNeedBytes(taskId); + len += Serialization.getNeedBytes(columnNames.length); + for (int i = 0; i < columnNames.length; ++i) { + len += Serialization.getNeedBytes(columnNames[i]); + } + len += 1; /*status*/ + len += Serialization.getNeedBytes(errorCode); + return len; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadCommitArg.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadCommitArg.java new file mode 100644 index 00000000..2a17c017 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadCommitArg.java @@ -0,0 +1,92 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadCommitArg, +// table_id_, +// task_id_); + +public class ObTableDirectLoadCommitArg implements ObSimplePayload { + + private long tableId = 0; + private long taskId = 0; + + public ObTableDirectLoadCommitArg() { + } + + public long getTableId() { + return tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + } + + public long getTaskId() { + return taskId; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVi64(buf, tableId); + Serialization.encodeVi64(buf, taskId); + } + + /** + * Decode. + */ + @Override + public ObTableDirectLoadCommitArg decode(ByteBuf buf) { + tableId = Serialization.decodeVi64(buf); + taskId = Serialization.decodeVi64(buf); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + return Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(taskId); + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadGetStatusArg.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadGetStatusArg.java new file mode 100644 index 00000000..0f4f614b --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadGetStatusArg.java @@ -0,0 +1,92 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadGetStatusArg, +// table_id_, +// task_id_); + +public class ObTableDirectLoadGetStatusArg implements ObSimplePayload { + + private long tableId = 0; + private long taskId = 0; + + public ObTableDirectLoadGetStatusArg() { + } + + public long getTableId() { + return tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + } + + public long getTaskId() { + return taskId; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVi64(buf, tableId); + Serialization.encodeVi64(buf, taskId); + } + + /** + * Decode. + */ + @Override + public ObTableDirectLoadGetStatusArg decode(ByteBuf buf) { + tableId = Serialization.decodeVi64(buf); + taskId = Serialization.decodeVi64(buf); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + return Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(taskId); + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadGetStatusRes.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadGetStatusRes.java new file mode 100644 index 00000000..8c804e73 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadGetStatusRes.java @@ -0,0 +1,93 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadGetStatusRes, +// status_, +// error_code_); + +public class ObTableDirectLoadGetStatusRes implements ObSimplePayload { + + private ObTableLoadClientStatus status = ObTableLoadClientStatus.MAX_STATUS; + private int errorCode = ResultCodes.OB_SUCCESS.errorCode; + + public ObTableDirectLoadGetStatusRes() { + } + + public ObTableLoadClientStatus getStatus() { + return status; + } + + public void setStatus(ObTableLoadClientStatus status) { + this.status = status; + } + + public int getErrorCode() { + return errorCode; + } + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeI8(buf, status.getByteValue()); + Serialization.encodeVi32(buf, errorCode); + } + + /** + * Decode. + */ + @Override + public ObTableDirectLoadGetStatusRes decode(ByteBuf buf) { + status = ObTableLoadClientStatus.valueOf(Serialization.decodeI8(buf)); + errorCode = Serialization.decodeVi32(buf); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + return 1 /*status*/+ Serialization.getNeedBytes(errorCode); + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadHeartBeatArg.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadHeartBeatArg.java new file mode 100644 index 00000000..01d48b78 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadHeartBeatArg.java @@ -0,0 +1,92 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadHeartBeatArg, +// table_id_, +// task_id_); + +public class ObTableDirectLoadHeartBeatArg implements ObSimplePayload { + + private long tableId = 0; + private long taskId = 0; + + public ObTableDirectLoadHeartBeatArg() { + } + + public long getTableId() { + return tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + } + + public long getTaskId() { + return taskId; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVi64(buf, tableId); + Serialization.encodeVi64(buf, taskId); + } + + /** + * Decode. + */ + @Override + public ObTableDirectLoadHeartBeatArg decode(ByteBuf buf) { + tableId = Serialization.decodeVi64(buf); + taskId = Serialization.decodeVi64(buf); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + return Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(taskId); + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadHeartBeatRes.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadHeartBeatRes.java new file mode 100644 index 00000000..11f7cd40 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadHeartBeatRes.java @@ -0,0 +1,93 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadHeartBeatRes, +// status_, +// error_code_); + +public class ObTableDirectLoadHeartBeatRes implements ObSimplePayload { + + private ObTableLoadClientStatus status = ObTableLoadClientStatus.MAX_STATUS; + private int errorCode = ResultCodes.OB_SUCCESS.errorCode; + + public ObTableDirectLoadHeartBeatRes() { + } + + public ObTableLoadClientStatus getStatus() { + return status; + } + + public void setStatus(ObTableLoadClientStatus status) { + this.status = status; + } + + public int getErrorCode() { + return errorCode; + } + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeI8(buf, status.getByteValue()); + Serialization.encodeVi32(buf, errorCode); + } + + /** + * Decode. + */ + @Override + public ObTableDirectLoadHeartBeatRes decode(ByteBuf buf) { + status = ObTableLoadClientStatus.valueOf(Serialization.decodeI8(buf)); + errorCode = Serialization.decodeVi32(buf); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + return 1 /*status*/+ Serialization.getNeedBytes(errorCode); + } + +} \ No newline at end of file diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadInsertArg.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadInsertArg.java new file mode 100644 index 00000000..453a1719 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadInsertArg.java @@ -0,0 +1,109 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.ObBytesString; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER_SIMPLE(ObTableDirectLoadInsertArg, +// table_id_, +// task_id_, +// payload_); + +public class ObTableDirectLoadInsertArg implements ObSimplePayload { + + private long tableId = 0; + private long taskId = 0; + private ObBytesString payload = new ObBytesString(); + + public ObTableDirectLoadInsertArg() { + } + + public long getTableId() { + return tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + } + + public long getTaskId() { + return taskId; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + public ObBytesString getPayload() { + return payload; + } + + public void setPayload(ObBytesString payload) { + if (payload == null) { + throw new NullPointerException(); + } + this.payload = payload; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVi64(buf, tableId); + Serialization.encodeVi64(buf, taskId); + Serialization.encodeBytesString(buf, payload); + } + + /** + * Decode. + */ + @Override + public ObTableDirectLoadInsertArg decode(ByteBuf buf) { + tableId = Serialization.decodeVi64(buf); + taskId = Serialization.decodeVi64(buf); + payload = Serialization.decodeBytesString(buf); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + return Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(taskId) + + Serialization.getNeedBytes(payload); + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadOperationType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadOperationType.java new file mode 100644 index 00000000..17022185 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadOperationType.java @@ -0,0 +1,52 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import java.util.HashMap; +import java.util.Map; + +public enum ObTableDirectLoadOperationType { + + BEGIN(0), COMMIT(1), ABORT(2), GET_STATUS(3), INSERT(4), HEART_BEAT(5), MAX_TYPE(6); + + private final int value; + private static final Map map = new HashMap(); + + static { + for (ObTableDirectLoadOperationType type : ObTableDirectLoadOperationType.values()) { + map.put(type.value, type); + } + } + + public static ObTableDirectLoadOperationType valueOf(int value) { + return map.get(value); + } + + ObTableDirectLoadOperationType(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + + public byte getByteValue() { + return (byte) value; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadRequest.java new file mode 100644 index 00000000..a5079126 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadRequest.java @@ -0,0 +1,195 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Credentialable; +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObAddr; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.ObBytesString; +import com.alipay.oceanbase.rpc.util.Serialization; +import com.alipay.oceanbase.rpc.util.TraceUtil; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER(ObTableDirectLoadRequest, +// header_, +// credential_, +// arg_content_); + +public class ObTableDirectLoadRequest extends AbstractPayload implements Credentialable { + + private Header header = new Header(); + private ObBytesString credential; + private ObBytesString argContent = new ObBytesString(); + + public ObTableDirectLoadRequest() { + setVersion(2); + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + public Header getHeader() { + return header; + } + + public void setHeader(Header header) { + this.header = header; + } + + public ObBytesString getCredential() { + return credential; + } + + @Override + public void setCredential(ObBytesString credential) { + this.credential = credential; + } + + public ObBytesString getArgContent() { + return argContent; + } + + public void setArgContent(ObBytesString argContent) { + if (argContent == null) { + throw new NullPointerException(); + } + this.argContent = argContent; + } + + public String toString() { + return String.format("%s {header:%s, argContent:%d}", TraceUtil.formatTraceMessage(this), + header, argContent.length()); + } + + /* + * Get pcode. + */ + @Override + public int getPcode() { + return Pcodes.OB_TABLE_API_DIRECT_LOAD; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + long payloadContentSize = getPayloadContentSize(); + int needBytes = (int) (Serialization.getObUniVersionHeaderLength(getVersion(), + payloadContentSize) + payloadContentSize); + ObByteBuf buf = new ObByteBuf(needBytes); + Serialization.encodeObUniVersionHeader(buf, getVersion(), payloadContentSize); + header.encode(buf); + Serialization.encodeBytesString(buf, credential); + Serialization.encodeBytesString(buf, argContent); + return buf.bytes; + } + + /** + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + header.decode(buf); + credential = Serialization.decodeBytesString(buf); + argContent = Serialization.decodeBytesString(buf); + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + return header.getEncodedSize() + Serialization.getNeedBytes(credential) + + Serialization.getNeedBytes(argContent); + } + + public static class Header implements ObSimplePayload { + private ObAddr addr = new ObAddr(); + private ObTableDirectLoadOperationType operationType = ObTableDirectLoadOperationType.MAX_TYPE; + + public Header() { + } + + public ObAddr getAddr() { + return addr; + } + + public void setAddr(ObAddr addr) { + this.addr = addr; + } + + public ObTableDirectLoadOperationType getOperationType() { + return operationType; + } + + public void setOperationType(ObTableDirectLoadOperationType operationType) { + this.operationType = operationType; + } + + public String toString() { + return String.format("{addr:%s, operationType:%s}", addr, operationType); + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + addr.encode(buf); + Serialization.encodeI8(buf, operationType.getByteValue()); + } + + /** + * Decode. + */ + @Override + public Header decode(ByteBuf buf) { + addr.decode(buf); + operationType = ObTableDirectLoadOperationType.valueOf(Serialization.decodeI8(buf)); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + return addr.getEncodedSize() + 1; + } + + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadResult.java new file mode 100644 index 00000000..bf22476e --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableDirectLoadResult.java @@ -0,0 +1,169 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObAddr; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.ObBytesString; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +// OB_SERIALIZE_MEMBER(ObTableDirectLoadResult, +// header_, +// res_content_); + +public class ObTableDirectLoadResult extends AbstractPayload { + + private Header header = new Header(); + private ObBytesString resContent = new ObBytesString(); + + public ObTableDirectLoadResult() { + setVersion(2); + } + + public Header getHeader() { + return header; + } + + public void setHeader(Header header) { + this.header = header; + } + + public ObBytesString getResContent() { + return resContent; + } + + public void setResContent(ObBytesString resContent) { + if (resContent == null) { + throw new NullPointerException(); + } + this.resContent = resContent; + } + + /* + * Get pcode. + */ + @Override + public int getPcode() { + return Pcodes.OB_TABLE_API_DIRECT_LOAD; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + long payloadContentSize = getPayloadContentSize(); + int needBytes = (int) (Serialization.getObUniVersionHeaderLength(getVersion(), + payloadContentSize) + payloadContentSize); + ObByteBuf buf = new ObByteBuf(needBytes); + Serialization.encodeObUniVersionHeader(buf, getVersion(), payloadContentSize); + header.encode(buf); + Serialization.encodeBytesString(buf, resContent); + return buf.bytes; + } + + /** + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + super.decode(buf); + header.decode(buf); + resContent = Serialization.decodeBytesString(buf); + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + return header.getEncodedSize() + Serialization.getNeedBytes(resContent); + } + + public static class Header implements ObSimplePayload { + + private ObAddr addr = new ObAddr(); + private ObTableDirectLoadOperationType operationType = ObTableDirectLoadOperationType.MAX_TYPE; + + public Header() { + } + + public ObAddr getAddr() { + return addr; + } + + public void setAddr(ObAddr addr) { + this.addr = addr; + } + + public ObTableDirectLoadOperationType getOperationType() { + return operationType; + } + + public void setOperationType(ObTableDirectLoadOperationType operationType) { + this.operationType = operationType; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = (int) getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + addr.encode(buf); + Serialization.encodeI8(buf, operationType.getByteValue()); + } + + /** + * Decode. + */ + @Override + public Header decode(ByteBuf buf) { + addr = new ObAddr(); + addr.decode(buf); + operationType = ObTableDirectLoadOperationType.valueOf(Serialization.decodeI8(buf)); + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + return addr.getEncodedSize() + 1; + } + + } + +} \ No newline at end of file diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableLoadClientStatus.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableLoadClientStatus.java new file mode 100644 index 00000000..35f6cdd0 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/direct_load/ObTableLoadClientStatus.java @@ -0,0 +1,52 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load; + +import java.util.HashMap; +import java.util.Map; + +public enum ObTableLoadClientStatus { + + RUNNING(0), COMMITTING(1), COMMIT(2), ERROR(3), ABORT(4), MAX_STATUS(5); + + private final int value; + private static final Map map = new HashMap(); + + static { + for (ObTableLoadClientStatus type : ObTableLoadClientStatus.values()) { + map.put(type.value, type); + } + } + + public static ObTableLoadClientStatus valueOf(int value) { + return map.get(value); + } + + ObTableLoadClientStatus(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + + public byte getByteValue() { + return (byte) value; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/AbstractObTableEntity.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/AbstractObTableEntity.java index e290cb6f..654d7ae6 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/AbstractObTableEntity.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/AbstractObTableEntity.java @@ -27,7 +27,7 @@ import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; -/** +/* * OB_DEF_SERIALIZE(ObITableEntity) @@ -183,7 +183,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -208,7 +208,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload size. */ @Override @@ -218,7 +218,7 @@ public long getPayloadSize() { } - /** + /* * Get payload content size. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObITableEntity.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObITableEntity.java index 80938be6..7344a030 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObITableEntity.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObITableEntity.java @@ -23,7 +23,7 @@ import java.util.Map; -/** +/* * OB_DEF_SERIALIZE(ObITableEntity) @@ -155,7 +155,7 @@ public interface ObITableEntity extends ObPayload { */ long getRowKeySize(); - /** + /* * Set rowkey size */ void setRowKeySize(long rowKeySize); diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObIndexType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObIndexType.java new file mode 100644 index 00000000..9b766c36 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObIndexType.java @@ -0,0 +1,74 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import java.util.HashMap; +import java.util.Map; + +public enum ObIndexType { + IndexTypeIsNot(0), IndexTypeNormalLocal(1), IndexTypeUniqueLocal(2), IndexTypeNormalGlobal(3), IndexTypeUniqueGlobal( + 4), IndexTypePrimary( + 5), IndexTypeDomainCtxcat( + 6), IndexTypeNormalGlobalLocalStorage( + 7), IndexTypeUniqueGlobalLocalStorage( + 8), IndexTypeSpatialLocal( + 10), IndexTypeSpatialGlobal( + 11), IndexTypeSpatialGlobalLocalStorage( + 12), IndexTypeMax( + 13); + + private int value; + private static Map map = new HashMap(); + + ObIndexType(int value) { + this.value = value; + } + + static { + for (ObIndexType type : ObIndexType.values()) { + map.put(type.value, type); + } + } + + /* + * Value of. + */ + public static ObIndexType valueOf(int value) { + return map.get(value); + } + + /* + * Get value. + */ + public int getValue() { + return value; + } + + /* + * Get byte value. + */ + public byte getByteValue() { + return (byte) value; + } + + public boolean isGlobalIndex() { + return valueOf(value) == ObIndexType.IndexTypeNormalGlobal + || valueOf(value) == ObIndexType.IndexTypeUniqueGlobal + || valueOf(value) == ObIndexType.IndexTypeSpatialGlobal; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableAbstractOperationRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableAbstractOperationRequest.java index 1c1e135f..41bba063 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableAbstractOperationRequest.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableAbstractOperationRequest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; import com.alipay.oceanbase.rpc.protocol.payload.Constants; import com.alipay.oceanbase.rpc.protocol.payload.Credentialable; @@ -32,29 +33,25 @@ public abstract class ObTableAbstractOperationRequest extends AbstractPayload im protected ObBytesString credential; // the credential returned when login. 登陆时返回的证书 protected String tableName; // table name. 待访问的表名 protected long tableId = Constants.OB_INVALID_ID; // table id. 如果知道表id，可以用于优化，如果不知道，设定为OB_INVALID_ID - protected long partitionId = Constants.OB_INVALID_ID; // Constants.OB_INVALID_ID; // partition id. 如果知道表分区id，可以用于优化，如果不知道，设定为OB_INVALID_ID - protected ObTableEntityType entityType = ObTableEntityType.DYNAMIC; // entity type. 如果明确entity类型，可以用于优化，如果不知道，设定为ObTableEntityType::DYNAMIC + protected long partitionId = Constants.INVALID_TABLET_ID; // Constants.OB_INVALID_ID; // partition id / tabletId. 如果知道表分区id，可以用于优化，如果不知道，设定为OB_INVALID_ID + protected ObTableEntityType entityType = ObTableEntityType.KV ; // entity type. 如果明确entity类型，可以用于优化，如果不知道，设定为ObTableEntityType::DYNAMIC protected ObTableConsistencyLevel consistencyLevel = ObTableConsistencyLevel.STRONG; // read consistency level. 读一致性，是否要强一致性等（必须读到刚写入的数据）. 目前只支持STRONG. - protected boolean returningRowKey = false; + protected ObTableOptionFlag option_flag = ObTableOptionFlag.DEFAULT; protected boolean returningAffectedEntity = false; protected boolean returningAffectedRows = false; - /** + /* * Get payload content size. */ @Override public long getPayloadContentSize() { - return Serialization.getNeedBytes(credential) + Serialization.getNeedBytes(tableName) - + Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(partitionId) + 2 - + 3; - } - - protected int encodeHeader(byte[] bytes, int idx) { - int headerLen = (int) getObUniVersionHeaderLength(getVersion(), getPayloadContentSize()); - System.arraycopy(encodeObUniVersionHeader(getVersion(), getPayloadContentSize()), 0, bytes, - idx, headerLen); - idx += headerLen; - return idx; + if (ObGlobal.obVsnMajor() >= 4) + return Serialization.getNeedBytes(credential) + Serialization.getNeedBytes(tableName) + + Serialization.getNeedBytes(tableId) + 8 + 2 + 3; + else + return Serialization.getNeedBytes(credential) + Serialization.getNeedBytes(tableName) + + Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(partitionId) + + 2 + 3; } protected int encodeCredential(byte[] bytes, int idx) { @@ -64,6 +61,9 @@ protected int encodeCredential(byte[] bytes, int idx) { return idx; } + /* + * tabletId also be treated as patitionId. + */ protected int encodeTableMetaWithPartitionId(byte[] bytes, int idx) { byte[] strbytes = Serialization.encodeVString(tableName); System.arraycopy(strbytes, 0, bytes, idx, strbytes.length); @@ -71,9 +71,14 @@ protected int encodeTableMetaWithPartitionId(byte[] bytes, int idx) { int len = Serialization.getNeedBytes(tableId); System.arraycopy(Serialization.encodeVi64(tableId), 0, bytes, idx, len); idx += len; - len = Serialization.getNeedBytes(partitionId); - System.arraycopy(Serialization.encodeVi64(partitionId), 0, bytes, idx, len); - idx += len; + if (ObGlobal.obVsnMajor() >= 4) { + System.arraycopy(Serialization.encodeI64(partitionId), 0, bytes, idx, 8); + idx += 8; + } else { + len = Serialization.getNeedBytes(partitionId); + System.arraycopy(Serialization.encodeVi64(partitionId), 0, bytes, idx, len); + idx += len; + } System.arraycopy(Serialization.encodeI8(entityType.getByteValue()), 0, bytes, idx, 1); idx += 1; return idx; @@ -97,28 +102,28 @@ protected int encodeConsistencyLevel(byte[] bytes, int idx) { return idx; } - /** + /* * Get credential. */ public ObBytesString getCredential() { return credential; } - /** + /* * Set timeout. */ public void setTimeout(long timeout) { this.timeout = timeout; } - /** + /* * Set tenant id. */ public void setTenantId(long tenantId) { this.tenantId = tenantId; } - /** + /* * Set credential. */ @Override @@ -126,112 +131,105 @@ public void setCredential(ObBytesString credential) { this.credential = credential; } - /** + /* * Get table name. */ public String getTableName() { return tableName; } - /** + /* * Set table name. */ public void setTableName(String tableName) { this.tableName = tableName; } - /** + /* * Get table id. */ public long getTableId() { return tableId; } - /** + /* * Set table id. */ public void setTableId(long tableId) { this.tableId = tableId; } - /** + /* * Get partition id. */ public long getPartitionId() { return partitionId; } - /** + /* * Set partition id. */ public void setPartitionId(long partitionId) { this.partitionId = partitionId; } - /** + /* * Get entity type. */ public ObTableEntityType getEntityType() { return entityType; } - /** + /* * Set entity type. */ public void setEntityType(ObTableEntityType entityType) { this.entityType = entityType; } - /** + /* * Get consistency level. */ public ObTableConsistencyLevel getConsistencyLevel() { return consistencyLevel; } - /** + /* * Set consistency level. */ public void setConsistencyLevel(ObTableConsistencyLevel consistencyLevel) { this.consistencyLevel = consistencyLevel; } - /** - * Is returning row key. - */ - public boolean isReturningRowKey() { - return returningRowKey; - } - - /** - * Set returning row key. + /* + * Set option flag. */ - public void setReturningRowKey(boolean returningRowKey) { - this.returningRowKey = returningRowKey; + public void setOptionFlag(ObTableOptionFlag optionFlagflag) { + this.option_flag = optionFlagflag; } - /** + /* * Is returning affected entity. */ public boolean isReturningAffectedEntity() { return returningAffectedEntity; } - /** + /* * Set returning affected entity. */ public void setReturningAffectedEntity(boolean returningAffectedEntity) { this.returningAffectedEntity = returningAffectedEntity; } - /** + /* * Is returning affected rows. */ public boolean isReturningAffectedRows() { return returningAffectedRows; } - /** + /* * Set returning affected rows. */ public void setReturningAffectedRows(boolean returningAffectedRows) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java index 61417172..84b03df6 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperation.java @@ -18,28 +18,27 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; -/** - * - OB_SERIALIZE_MEMBER(ObTableOperation, operation_type_, const_cast(*entity_)); +/* + * OB_SERIALIZE_MEMBER(ObTableOperation, operation_type_, const_cast(*entity_)); * */ public class ObTableBatchOperation extends AbstractPayload { - private List tableOperations = new ArrayList(); - private boolean isReadOnly = true; - private boolean isSameType; - private boolean isSamePropertiesNames; + private List tableOperations = new ArrayList(); + private boolean isReadOnly = true; + private boolean isSameType = true; + private boolean isSamePropertiesNames = true; - /** + /* * Encode. */ @Override @@ -76,7 +75,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -101,7 +100,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -115,73 +114,128 @@ public long getPayloadContentSize() { return payloadContentSize + 3; } - /** + /* * Get table operations. */ public List getTableOperations() { return tableOperations; } - /** + /* + * hash_map keys to TreeSet IgnoringCase + */ + public TreeSet mapKeysToSetIgnoringCase(Set keys) { + TreeSet keySetIgnoreCase = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + keySetIgnoreCase.addAll(keys); + return keySetIgnoreCase; + } + + /* * Add table operation. */ public void addTableOperation(ObTableOperation tableOperation) { this.tableOperations.add(tableOperation); + int length = this.tableOperations.size(); if (isReadOnly && !tableOperation.isReadonly()) { isReadOnly = false; } + if (isSameType + && length > 1 + && tableOperations.get(length - 1).getOperationType() != tableOperations + .get(length - 2).getOperationType()) { + isSameType = false; + } + // 判断是否是 same_properties_name + if (isSamePropertiesNames && length > 1) { + ObTableOperation prev = tableOperations.get(length - 2); + ObTableOperation curr = tableOperations.get(length - 1); + if (prev.getEntity() == null || curr.getEntity() == null) { + isSamePropertiesNames = false; + } else if (prev.getEntity().getPropertiesCount() != curr.getEntity() + .getPropertiesCount()) { + isSamePropertiesNames = false; + } else { + isSamePropertiesNames = mapKeysToSetIgnoringCase( + prev.getEntity().getProperties().keySet()).equals( + mapKeysToSetIgnoringCase(curr.getEntity().getProperties().keySet())); + } + } } - /** + /* * Set table operations. */ public void setTableOperations(List tableOperations) { this.tableOperations = tableOperations; this.isReadOnly = true; + this.isSameType = true; + this.isSamePropertiesNames = true; + ObTableOperationType prevType = null; + TreeSet firstKeySetIgnoreCase = null; for (ObTableOperation o : tableOperations) { - if (!o.isReadonly()) { - this.isReadOnly = false; + if (this.isReadOnly || this.isSameType || this.isSamePropertiesNames) { + if (!o.isReadonly()) { + this.isReadOnly = false; + } + if (prevType != null && prevType != o.getOperationType()) { + this.isSameType = false; + } else { + prevType = o.getOperationType(); + } + + if (this.isSamePropertiesNames) { + if (firstKeySetIgnoreCase == null) { + firstKeySetIgnoreCase = mapKeysToSetIgnoringCase(o.getEntity() + .getProperties().keySet()); + } else if (firstKeySetIgnoreCase.size() != o.getEntity().getPropertiesCount()) { + this.isSamePropertiesNames = false; + } else { + this.isSamePropertiesNames = firstKeySetIgnoreCase + .equals(mapKeysToSetIgnoringCase(o.getEntity().getProperties().keySet())); + } + } + } else { return; } } } - /** + /* * Is read only. */ public boolean isReadOnly() { return isReadOnly; } - /** + /* * Set read only. */ public void setReadOnly(boolean readOnly) { isReadOnly = readOnly; } - /** + /* * Is same type. */ public boolean isSameType() { return isSameType; } - /** + /* * Set same type. */ public void setSameType(boolean sameType) { isSameType = sameType; } - /** + /* * Is same properties names. */ public boolean isSamePropertiesNames() { return isSamePropertiesNames; } - /** + /* * Set same properties names. */ public void setSamePropertiesNames(boolean samePropertiesNames) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java index e92deec8..25eeca53 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationRequest.java @@ -17,11 +17,12 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; -/** +/* * OB_SERIALIZE_MEMBER(ObTableBatchOperationRequest, credential_, @@ -43,7 +44,7 @@ public class ObTableBatchOperationRequest extends ObTableAbstractOperationReques private ObTableBatchOperation batchOperation; private boolean batchOperationAsAtomic = false; - /** + /* * Get pcode. */ @Override @@ -51,7 +52,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_BATCH_EXECUTE; } - /** + /* * Encode. */ @Override @@ -75,8 +76,7 @@ public byte[] encode() { idx += len; System.arraycopy(Serialization.encodeI8(consistencyLevel.getByteValue()), 0, bytes, idx, 1); idx++; - System.arraycopy(Serialization.encodeI8(returningRowKey ? (byte) 1 : (byte) 0), 0, bytes, - idx, 1); + System.arraycopy(Serialization.encodeI8(option_flag.getByteValue()), 0, bytes, idx, 1); idx++; System.arraycopy(Serialization.encodeI8(returningAffectedEntity ? (byte) 1 : (byte) 0), 0, bytes, idx, 1); @@ -84,17 +84,22 @@ public byte[] encode() { System.arraycopy(Serialization.encodeI8(returningAffectedRows ? (byte) 1 : (byte) 0), 0, bytes, idx, 1); idx++; - len = Serialization.getNeedBytes(partitionId); - System.arraycopy(Serialization.encodeVi64(partitionId), 0, bytes, idx, len); + if (ObGlobal.obVsnMajor() >= 4) { + System.arraycopy(Serialization.encodeI64(partitionId), 0, bytes, idx, 8); + idx += 8; + } else { + len = Serialization.getNeedBytes(partitionId); + System.arraycopy(Serialization.encodeVi64(partitionId), 0, bytes, idx, len); + idx += len; + } - idx += len; System.arraycopy(Serialization.encodeI8(batchOperationAsAtomic ? (byte) 1 : (byte) 0), 0, bytes, idx, 1); return bytes; } - /** + /* * Decode. */ @Override @@ -109,16 +114,19 @@ public Object decode(ByteBuf buf) { this.batchOperation.decode(buf); this.consistencyLevel = ObTableConsistencyLevel.valueOf(buf.readByte()); - this.returningRowKey = Serialization.decodeI8(buf) != 0; + this.option_flag = ObTableOptionFlag.valueOf(buf.readByte()); this.returningAffectedEntity = Serialization.decodeI8(buf) != 0; this.returningAffectedRows = Serialization.decodeI8(buf) != 0; - this.partitionId = Serialization.decodeVi64(buf); + if (ObGlobal.obVsnMajor() >= 4) + this.partitionId = Serialization.decodeI64(buf); + else + this.partitionId = Serialization.decodeVi64(buf); this.batchOperationAsAtomic = Serialization.decodeI8(buf) != 0; return this; } - /** + /* * Get payload content size. */ @Override @@ -126,31 +134,39 @@ public long getPayloadContentSize() { return super.getPayloadContentSize() + batchOperation.getPayloadSize() + 1; } - /** + /* * Get batch operation. */ public ObTableBatchOperation getBatchOperation() { return batchOperation; } - /** + /* * Set batch operation. */ public void setBatchOperation(ObTableBatchOperation batchOperation) { this.batchOperation = batchOperation; } - /** + /* * Whether BatchOperation As Atomic. */ public boolean isBatchOperationAsAtomic() { return batchOperationAsAtomic; } - /** + /* * Set BatchOperation As Atomic. */ public void setBatchOperationAsAtomic(boolean batchOperationAsAtomic) { this.batchOperationAsAtomic = batchOperationAsAtomic; } + + public void setBatchOpReturnOneResult(boolean returnOneResult) { + if (returnOneResult == true) { + this.option_flag.setReturnOneResult(true); + } else { + this.option_flag.setReturnOneResult(false); + } + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationResult.java index 6ca6a126..8f00cad9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationResult.java @@ -32,7 +32,7 @@ public class ObTableBatchOperationResult extends AbstractPayload { private List results = new ArrayList(); - /** + /* * Get pcode. */ @Override @@ -40,7 +40,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_BATCH_EXECUTE; } - /** + /* * Encode. */ @Override @@ -69,7 +69,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -89,7 +89,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -103,32 +103,31 @@ public long getPayloadContentSize() { return payloadContentSize; } - /** + /* * Get results. */ public List getResults() { return results; } - /** + /* * Set results. */ public void setResults(List results) { this.results = results; } - /** + /* * Add result. */ public void addResult(ObTableOperationResult result) { this.results.add(result); } - /** + /* * Add all results. */ public void addAllResults(List results) { this.results.addAll(results); } - } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableConsistencyLevel.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableConsistencyLevel.java index 85ff0812..b2056fa5 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableConsistencyLevel.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableConsistencyLevel.java @@ -37,21 +37,21 @@ public enum ObTableConsistencyLevel { } } - /** + /* * Value of. */ public static ObTableConsistencyLevel valueOf(int value) { return map.get(value); } - /** + /* * Get value. */ public int getValue() { return value; } - /** + /* * Get byte value. */ public byte getByteValue() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableEntity.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableEntity.java index 23945b3f..47339289 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableEntity.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableEntity.java @@ -28,7 +28,7 @@ public class ObTableEntity extends AbstractObTableEntity { private ObRowKey rowKey = new ObRowKey(); private Map properties = new HashMap(); - /** + /* * Set row key value. */ @Override @@ -36,7 +36,7 @@ public void setRowKeyValue(long idx, ObObj rowKeyValue) { rowKey.setObj((int) idx, rowKeyValue); } - /** + /* * Add row key value. */ @Override @@ -44,7 +44,7 @@ public void addRowKeyValue(ObObj rowKeyValue) { rowKey.addObj(rowKeyValue); } - /** + /* * Get row key value. */ @Override @@ -52,7 +52,7 @@ public ObObj getRowKeyValue(long idx) { return rowKey.getObj((int) idx); } - /** + /* * Get row key size. */ @Override @@ -60,7 +60,7 @@ public long getRowKeySize() { return rowKey.getObjCount(); } - /** + /* * Set row key size. */ @Override @@ -68,7 +68,7 @@ public void setRowKeySize(long rowKeySize) { // ignore } - /** + /* * Get row key. */ @Override @@ -76,7 +76,7 @@ public ObRowKey getRowKey() { return rowKey; } - /** + /* * Set property. */ @Override @@ -84,7 +84,7 @@ public void setProperty(String propName, ObObj propValue) { this.properties.put(propName, propValue); } - /** + /* * Get property. */ @Override @@ -105,7 +105,7 @@ public Map getSimpleProperties() { return values; } - /** + /* * Get properties count. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableEntityType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableEntityType.java index 8c34b6a7..aee6bee9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableEntityType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableEntityType.java @@ -37,21 +37,21 @@ public enum ObTableEntityType { } } - /** + /* * Value of. */ public static ObTableEntityType valueOf(int value) { return map.get(value); } - /** + /* * Get value. */ public int getValue() { return value; } - /** + /* * Get byte value. */ public byte getByteValue() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpFlag.java new file mode 100644 index 00000000..19f73133 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpFlag.java @@ -0,0 +1,78 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +public class ObTableLSOpFlag { + private static final int FLAG_IS_SAME_TYPE = 1 << 0; + private static final int FLAG_IS_SAME_PROPERTIES_NAMES = 1 << 1; + private static final int FLAG_RETURN_ONE_RESULT = 1 << 2; + private static final int FLAG_NEED_ALL_PROP = 1 << 3; + private long flags = 0; + + public void setFlagIsSameType(boolean isSameType) { + if (isSameType) { + flags |= FLAG_IS_SAME_TYPE; + } else { + flags &= ~FLAG_IS_SAME_TYPE; + } + } + + public void setFlagIsSamePropertiesNames(boolean isSamePropertiesNames) { + if (isSamePropertiesNames) { + flags |= FLAG_IS_SAME_PROPERTIES_NAMES; + } else { + flags &= ~FLAG_IS_SAME_PROPERTIES_NAMES; + } + } + + public void setReturnOneResult(boolean returnOneResult) { + if (returnOneResult) { + flags |= FLAG_RETURN_ONE_RESULT; + } else { + flags &= ~FLAG_RETURN_ONE_RESULT; + } + } + + public void setFlagNeedAllProp(boolean needAllProp) { + if (needAllProp) { + flags |= FLAG_NEED_ALL_PROP; + } else { + flags &= ~FLAG_NEED_ALL_PROP; + } + } + + public long getValue() { + return flags; + } + + public void setValue(long flags) { + this.flags = flags; + } + + public boolean getFlagIsSameType() { + return (flags & FLAG_IS_SAME_TYPE) != 0; + } + + public boolean getFlagIsSamePropertiesNames() { + return (flags & FLAG_IS_SAME_PROPERTIES_NAMES) != 0; + } + + public boolean getFlagNeedAllProp() { + return (flags & FLAG_NEED_ALL_PROP) != 0; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpRequest.java new file mode 100644 index 00000000..48b5e1c0 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpRequest.java @@ -0,0 +1,156 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Credentialable; +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.util.ObBytesString; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; +import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; + +/* +OB_SERIALIZE_MEMBER(ObTableLSOpRequest, + credential_, + entity_type_, + consistency_level_, + ls_op_); + */ +public class ObTableLSOpRequest extends AbstractPayload implements Credentialable { + protected ObBytesString credential; + protected ObTableEntityType entityType = ObTableEntityType.KV; + protected ObTableConsistencyLevel consistencyLevel = ObTableConsistencyLevel.STRONG; + private ObTableLSOperation lsOperation = null; + + /* + * Get pcode. + */ + @Override + public int getPcode() { + return Pcodes.OB_TABLE_API_LS_EXECUTE; + } + + /* + * Encode. + */ + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode ObTableLSOpRequest header + idx = encodeHeader(bytes, idx); + + // 1. encode credential + byte[] strbytes = Serialization.encodeBytesString(credential); + System.arraycopy(strbytes, 0, bytes, idx, strbytes.length); + idx += strbytes.length; + + // 2. encode entity_type + System.arraycopy(Serialization.encodeI8(entityType.getByteValue()), 0, bytes, idx, 1); + idx++; + + // 3. encode consistencyLevel level + System.arraycopy(Serialization.encodeI8(consistencyLevel.getByteValue()), 0, bytes, idx, 1); + idx++; + + // 4. encode lsOperation + int len = (int) lsOperation.getPayloadSize(); + System.arraycopy(lsOperation.encode(), 0, bytes, idx, len); + idx += len; + + return bytes; + } + + /* + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + super.decode(buf); + this.credential = Serialization.decodeBytesString(buf); + this.entityType = ObTableEntityType.valueOf(buf.readByte()); + this.consistencyLevel = ObTableConsistencyLevel.valueOf(buf.readByte()); + this.lsOperation = new ObTableLSOperation(); + this.lsOperation.decode(buf); + + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + return lsOperation.getPayloadSize() + Serialization.getNeedBytes(credential) + 1 // entityType + + 1; // consistencyLevel + } + + /* + * Get batch operation. + */ + public ObTableLSOperation getLSOperation() { + return lsOperation; + } + + /* + * Set batch operation. + */ + public void addTabletOperation(ObTableTabletOp tabletOp) { + lsOperation.addTabletOperation(tabletOp); + } + + public void setLsOperation(ObTableLSOperation lsOperation) { + this.lsOperation = lsOperation; + } + + /* + * Set entity type. + */ + public void setEntityType(ObTableEntityType entityType) { + this.entityType = entityType; + } + + /* + * Set timeout. + */ + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + /* + * Set consistency level. + */ + public void setConsistencyLevel(ObTableConsistencyLevel consistencyLevel) { + this.consistencyLevel = consistencyLevel; + } + + /* + * Set credential. + */ + public void setCredential(ObBytesString credential) { + this.credential = credential; + } + + public void setTableId(long tableId) { + this.lsOperation.setTableId(tableId); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpResult.java new file mode 100644 index 00000000..1305cad7 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOpResult.java @@ -0,0 +1,148 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import java.util.ArrayList; +import java.util.List; + +import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; +import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; + +public class ObTableLSOpResult extends AbstractPayload { + + private List results = new ArrayList(); + private List propertiesColumnNames = new ArrayList<>(); + + /* + * Get pcode. + */ + @Override + public int getPcode() { + return Pcodes.OB_TABLE_API_LS_EXECUTE; + } + + /* + * Encode. + */ + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode header + idx = encodeHeader(bytes, idx); + + // 1. encode columnNames + int len = Serialization.getNeedBytes(this.propertiesColumnNames.size()); + System.arraycopy(Serialization.encodeVi64(this.propertiesColumnNames.size()), 0, bytes, idx, len); + for (String columnName : propertiesColumnNames) { + len = Serialization.getNeedBytes(columnName); + System.arraycopy(Serialization.encodeVString(columnName), 0, bytes, idx, len); + idx += len; + } + + // 2. encode results + len = Serialization.getNeedBytes(this.results.size()); + System.arraycopy(Serialization.encodeVi64(this.results.size()), 0, bytes, idx, len); + idx += len; + + for (ObTableTabletOpResult result : results) { + len = (int) result.getPayloadSize(); + System.arraycopy(result.encode(), 0, bytes, idx, len); + idx += len; + } + + return bytes; + } + + /* + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + // 0. decode version + super.decode(buf); + + // 1. decode column names + int len = (int) Serialization.decodeVi64(buf); + for (int i = 0; i < len; i++) { + String column = Serialization.decodeVString(buf); + propertiesColumnNames.add(column); + } + + // 2. decode results + len = (int) Serialization.decodeVi64(buf); + results = new ArrayList(len); + for (int i = 0; i < len; i++) { + ObTableTabletOpResult tabletOpResult = new ObTableTabletOpResult(); + tabletOpResult.setPropertiesColumnNames(this.propertiesColumnNames); + tabletOpResult.decode(buf); + results.add(tabletOpResult); + } + + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + long payloadContentSize = 0; + payloadContentSize += Serialization.getNeedBytes(results.size()); + for (ObTableTabletOpResult result : results) { + payloadContentSize += result.getPayloadSize(); + } + + return payloadContentSize; + } + + /* + * Get results. + */ + public List getResults() { + return results; + } + + /* + * Set results. + */ + public void setResults(List results) { + this.results = results; + } + + /* + * Add result. + */ + public void addResult(ObTableTabletOpResult result) { + this.results.add(result); + } + + /* + * Add all results. + */ + public void addAllResults(List results) { + this.results.addAll(results); + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOperation.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOperation.java new file mode 100644 index 00000000..86a52e61 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableLSOperation.java @@ -0,0 +1,344 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Constants; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import java.util.*; + +public class ObTableLSOperation extends AbstractPayload { + + private List tabletOperations = new ArrayList(); + private long lsId = INVALID_LS_ID; // i64 + + private String tableName; + + private long tableId = Constants.OB_INVALID_ID;; + + // common column names for all single operation + private List rowKeyNames = new ArrayList<>(); + private Set rowKeyNamesSet = new LinkedHashSet<>(); + private Map rowkeyColumnNamesIdxMap = new HashMap<>(); + + private List propertiesNames = new ArrayList<>(); + private Set propertiesNamesSet = new LinkedHashSet<>(); + private Map propertiesColumnNamesIdxMap = new HashMap<>(); + + private ObTableLSOpFlag optionFlag = new ObTableLSOpFlag(); + + private static final int LS_ID_SIZE = 8; + private static final long INVALID_LS_ID = -1; + + /* + OB_UNIS_DEF_SERIALIZE(ObTableLSOp, + ls_id_, + option_flag_, + tablet_ops_); + */ + + /* + * Encode. + */ + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode header + idx = encodeHeader(bytes, idx); + + // 1. encode ls id + System.arraycopy(Serialization.encodeI64(lsId), 0, bytes, idx, 8); + idx += 8; + + // 2. encode table name + int len = Serialization.getNeedBytes(tableName); + System.arraycopy(Serialization.encodeVString(tableName), 0, bytes, idx, len); + idx += len; + + // 3. encode table id + len = Serialization.getNeedBytes(tableId); + System.arraycopy(Serialization.encodeVi64(tableId), 0, bytes, idx, len); + idx += len; + + // 4. encode rowKey names + len = Serialization.getNeedBytes(rowKeyNames.size()); + System.arraycopy(Serialization.encodeVi64(rowKeyNames.size()), 0, bytes, idx, len); + idx += len; + for (String rowKeyName : rowKeyNames) { + len = Serialization.getNeedBytes(rowKeyName); + System.arraycopy(Serialization.encodeVString(rowKeyName), 0, bytes, idx, len); + idx += len; + } + + // 5. encode properties names + len = Serialization.getNeedBytes(propertiesNames.size()); + System.arraycopy(Serialization.encodeVi64(propertiesNames.size()), 0, bytes, idx, len); + idx += len; + for (String propertyName : propertiesNames) { + len = Serialization.getNeedBytes(propertyName); + System.arraycopy(Serialization.encodeVString(propertyName), 0, bytes, idx, len); + idx += len; + } + + // 6. encode option flag + len = Serialization.getNeedBytes(optionFlag.getValue()); + System.arraycopy(Serialization.encodeVi64(optionFlag.getValue()), 0, bytes, idx, len); + idx += len; + + // 7. encode Operation + len = Serialization.getNeedBytes(tabletOperations.size()); + System.arraycopy(Serialization.encodeVi64(tabletOperations.size()), 0, bytes, idx, len); + idx += len; + for (ObTableTabletOp tabletOperation : tabletOperations) { + len = (int) tabletOperation.getPayloadSize(); + System.arraycopy(tabletOperation.encode(), 0, bytes, idx, len); + idx += len; + } + + return bytes; + } + + /* + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + // 0. decode header + super.decode(buf); + + // 1. decode others + this.lsId = Serialization.decodeI64(buf); + + // 2. decode table name + this.tableName = Serialization.decodeVString(buf); + + // 3. decode table id + this.tableId = Serialization.decodeVi64(buf); + + // 4. decode rowKey names + int len = (int) Serialization.decodeVi64(buf); + for (int i = 0; i < len; i++) { + String rowkeyName = Serialization.decodeVString(buf); + rowKeyNames.add(rowkeyName); + } + + // 5. decode properties names + len = (int) Serialization.decodeVi64(buf); + for (int i = 0; i < len; i++) { + String propertyName = Serialization.decodeVString(buf); + propertiesNames.add(propertyName); + } + + // 6. decode flags + long flagValue = Serialization.decodeVi64(buf); + optionFlag.setValue(flagValue); + + // 7. decode Operation + len = (int) Serialization.decodeVi64(buf); + tabletOperations = new ArrayList(len); + for (int i = 0; i < len; i++) { + ObTableTabletOp tabletOperation = new ObTableTabletOp(); + tabletOperation.decode(buf); + tabletOperations.add(tabletOperation); + } + + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + long payloadContentSize = 0; + payloadContentSize += Serialization.getNeedBytes(tabletOperations.size()); + for (ObTableTabletOp operation : tabletOperations) { + payloadContentSize += operation.getPayloadSize(); + } + + payloadContentSize += Serialization.getNeedBytes(rowKeyNames.size()); + for (String rowKeyName : rowKeyNames) { + payloadContentSize += Serialization.getNeedBytes(rowKeyName); + } + + payloadContentSize += Serialization.getNeedBytes(propertiesNames.size()); + for (String propertyName : propertiesNames) { + payloadContentSize += Serialization.getNeedBytes(propertyName); + } + + return payloadContentSize + LS_ID_SIZE + Serialization.getNeedBytes(optionFlag.getValue()) + + Serialization.getNeedBytes(tableName) + Serialization.getNeedBytes(tableId); + } + + /* + * Get table operations. + */ + public List getTabletOperations() { + return tabletOperations; + } + + /* + * Add table operation. + */ + public void addTabletOperation(ObTableTabletOp tabletOperation) { + this.tabletOperations.add(tabletOperation); + int length = this.tabletOperations.size(); + + // set column names + this.rowKeyNamesSet.addAll(tabletOperation.getRowKeyNamesSet()); + this.propertiesNamesSet.addAll(tabletOperation.getPropertiesNamesSet()); + + if (length == 1 && tabletOperation.isSameType()) { + setIsSameType(true); + return; + } + + if (isSameType() + && length > 1 + && !(tabletOperation.isSameType() && (this.tabletOperations.get(length - 1) + .getSingleOperations().get(0).getSingleOpType() == this.tabletOperations + .get(length - 2).getSingleOperations().get(0).getSingleOpType()))) { + setIsSameType(false); + } + } + + public void setLsId(long lsId) { + this.lsId = lsId; + } + + public void setReturnOneResult(boolean returnOneResult) { + optionFlag.setReturnOneResult(returnOneResult); + } + + public boolean isSameType() { + return optionFlag.getFlagIsSameType(); + } + + public void setIsSameType(boolean isSameType) { + optionFlag.setFlagIsSameType(isSameType); + } + + public boolean isSamePropertiesNames() { + return optionFlag.getFlagIsSamePropertiesNames(); + } + + public void setIsSamePropertiesNames(boolean isSamePropertiesNames) { + optionFlag.setFlagIsSamePropertiesNames(isSamePropertiesNames); + } + + public void setNeedAllProp(boolean needAllProp) { optionFlag.setFlagNeedAllProp(needAllProp);} + + public boolean isNeedAllProp() { return optionFlag.getFlagNeedAllProp(); } + + public long getTableId() { + return tableId; + } + + public void setTableId(long tableId) { + this.tableId = tableId; + } + + public void prepareOption() { + // set isSamePropertiesNames into entities + if (isSamePropertiesNames()) { + for (ObTableTabletOp tabletOp : tabletOperations) { + for (ObTableSingleOp singleOp : tabletOp.getSingleOperations()) { + for (ObTableSingleOpEntity entity : singleOp.getEntities()) { + entity.setIgnoreEncodePropertiesColumnNames(true); + // todo: set other option in one loop + } + } + } + } + + // todo: set other option in other loop + } + + public void collectColumnNamesIdxMap() { + // prepare rowkey idx map + long index = 0; + for (String rowkeyName : rowKeyNamesSet) { + this.rowKeyNames.add(rowkeyName); + this.rowkeyColumnNamesIdxMap.put(rowkeyName, index); + index += 1; + } + + // prepare properties idx map + index = 0; + for (String propertiesName : propertiesNamesSet) { + this.propertiesNames.add(propertiesName); + this.propertiesColumnNamesIdxMap.put(propertiesName, index); + index += 1; + } + } + + /* + * beforeOption is used to collect necessary data from entity before prepareOption + */ + public void beforeOption() { + boolean isSamePropertiesColumnNames = true; + + for (ObTableTabletOp tabletOp : tabletOperations) { + for (ObTableSingleOp singleOp : tabletOp.getSingleOperations()) { + for (ObTableSingleOpEntity entity : singleOp.getEntities()) { + // if column names are the same, then the length should be the same + isSamePropertiesColumnNames = entity.isSamePropertiesColumnNamesLen(this.propertiesColumnNamesIdxMap.size()); + if (!isSamePropertiesColumnNames) break; + } + if (!isSamePropertiesColumnNames) break; + } + if (!isSamePropertiesColumnNames) break; + } + + if (isSamePropertiesColumnNames) this.setIsSamePropertiesNames(true); + } + + + public void prepareColumnNamesBitMap() { + // adjust query & entity + for (ObTableTabletOp tabletOp : tabletOperations) { + for (ObTableSingleOp singleOp : tabletOp.getSingleOperations()) { + singleOp.getQuery().adjustScanRangeColumns(rowkeyColumnNamesIdxMap); + for (ObTableSingleOpEntity entity : singleOp.getEntities()) { + entity.adjustRowkeyColumnName(rowkeyColumnNamesIdxMap); + entity.adjustPropertiesColumnName(propertiesColumnNamesIdxMap); + } + } + } + } + + public void prepare() { + this.collectColumnNamesIdxMap(); + this.beforeOption(); + this.prepareOption(); + this.prepareColumnNamesBitMap(); + } + + public long getLsId() { + return lsId; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperation.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperation.java index bc847e4d..5e439ef6 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperation.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperation.java @@ -27,7 +27,7 @@ import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; -/** +/* * OB_SERIALIZE_MEMBER(ObTableOperation, operation_type_, const_cast(*entity_)); * @@ -35,9 +35,9 @@ public class ObTableOperation extends AbstractPayload { private ObTableOperationType operationType; - private ObITableEntity entity; // TODO 我是如何知道类型的？ + private ObITableEntity entity; - /** + /* * Encode. */ @Override @@ -62,7 +62,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -80,7 +80,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -88,35 +88,35 @@ public long getPayloadContentSize() { return 1 + entity.getPayloadSize(); } - /** + /* * Get operation type. */ public ObTableOperationType getOperationType() { return operationType; } - /** + /* * Set operation type. */ public void setOperationType(ObTableOperationType operationType) { this.operationType = operationType; } - /** + /* * Get entity. */ public ObITableEntity getEntity() { return entity; } - /** + /* * Set entity. */ public void setEntity(ObITableEntity entity) { this.entity = entity; } - /** + /* * Get instance. */ public static ObTableOperation getInstance(ObTableOperationType type, Object[] rowKeys, @@ -127,14 +127,12 @@ public static ObTableOperation getInstance(ObTableOperationType type, Object[] r obTableOperation.setOperationType(type); ObITableEntity entity = new ObTableEntity(); obTableOperation.setEntity(entity); - for (int i = 0; i < rowKeys.length; i++) { - Object rowkey = rowKeys[i]; - ObObjMeta rowkeyMeta = ObObjType.defaultObjMeta(rowkey); - - ObObj obj = new ObObj(); - obj.setMeta(rowkeyMeta); - obj.setValue(rowkey); - entity.addRowKeyValue(obj); + if (rowKeys != null) { + for (int i = 0; i < rowKeys.length; i++) { + Object rowkey = rowKeys[i]; + ObObj obj = ObObj.getInstance(rowkey); + entity.addRowKeyValue(obj); + } } if (columns != null) { @@ -144,11 +142,7 @@ public static ObTableOperation getInstance(ObTableOperationType type, Object[] r if (properties != null) { value = properties[i]; } - ObObjMeta meta = ObObjType.defaultObjMeta(value); - - ObObj c = new ObObj(); - c.setMeta(meta); - c.setValue(value); + ObObj c = ObObj.getInstance(value); entity.setProperty(name, c); } } @@ -156,7 +150,7 @@ public static ObTableOperation getInstance(ObTableOperationType type, Object[] r return obTableOperation; } - /** + /* * Is readonly. */ public boolean isReadonly() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationRequest.java index dbc2496e..3fe9c97a 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationRequest.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationRequest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; @@ -41,7 +42,7 @@ public class ObTableOperationRequest extends ObTableAbstractOperationRequest { private ObTableOperation tableOperation; - /** + /* * Get pcode. */ @Override @@ -49,7 +50,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_EXECUTE; } - /** + /* * Encode. */ @Override @@ -70,8 +71,7 @@ public byte[] encode() { idx += len; System.arraycopy(Serialization.encodeI8(consistencyLevel.getByteValue()), 0, bytes, idx, 1); idx++; - System.arraycopy(Serialization.encodeI8(returningRowKey ? (byte) 1 : (byte) 0), 0, bytes, - idx, 1); + System.arraycopy(Serialization.encodeI8(option_flag.getByteValue()), 0, bytes, idx, 1); idx++; System.arraycopy(Serialization.encodeI8(returningAffectedEntity ? (byte) 1 : (byte) 0), 0, bytes, idx, 1); @@ -82,7 +82,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -92,7 +92,10 @@ public Object decode(ByteBuf buf) { this.credential = Serialization.decodeBytesString(buf); this.tableName = Serialization.decodeVString(buf); this.tableId = Serialization.decodeVi64(buf); - this.partitionId = Serialization.decodeVi64(buf); + if (ObGlobal.obVsnMajor() >= 4) + this.partitionId = Serialization.decodeI64(buf); + else + this.partitionId = Serialization.decodeVi64(buf); this.entityType = ObTableEntityType.valueOf(buf.readByte()); this.tableOperation = new ObTableOperation(); if (ObTableEntityType.DYNAMIC.equals(this.entityType)) { @@ -103,14 +106,14 @@ public Object decode(ByteBuf buf) { this.tableOperation.decode(buf); this.consistencyLevel = ObTableConsistencyLevel.valueOf(buf.readByte()); - this.returningRowKey = Serialization.decodeI8(buf) != 0; + this.option_flag = ObTableOptionFlag.valueOf(buf.readByte()); this.returningAffectedEntity = Serialization.decodeI8(buf) != 0; this.returningAffectedRows = Serialization.decodeI8(buf) != 0; return this; } - /** + /* * Get payload content size. */ @Override @@ -118,21 +121,21 @@ public long getPayloadContentSize() { return super.getPayloadContentSize() + tableOperation.getPayloadSize(); } - /** + /* * Get table operation. */ public ObTableOperation getTableOperation() { return tableOperation; } - /** + /* * Set table operation. */ public void setTableOperation(ObTableOperation tableOperation) { this.tableOperation = tableOperation; } - /** + /* * Get instance. */ public static ObTableOperationRequest getInstance(String tableName, // @@ -144,8 +147,7 @@ public static ObTableOperationRequest getInstance(String tableName, // ObTableOperationRequest request = new ObTableOperationRequest(); request.setTableName(tableName); - // us - request.setTimeout(timeout * 1000); + request.setTimeout(timeout); request.setReturningAffectedRows(true); // TODO 可以设置参数，如果不用的话提高性能 ObTableOperation obTableOperation = ObTableOperation.getInstance(type, rowKeys, columns, properties); diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationResult.java index 17d36b91..f8a50ee9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationResult.java @@ -31,7 +31,7 @@ public class ObTableOperationResult extends AbstractPayload { private String executeHost; private int executePort; - /** + /* * Get pcode. */ @Override @@ -39,7 +39,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_EXECUTE; } - /** + /* * Encode. */ @Override @@ -73,7 +73,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -87,7 +87,7 @@ public Object decode(ByteBuf buf) { // 2. decode itself this.operationType = ObTableOperationType.valueOf(Serialization.decodeI8(buf.readByte())); - // 3. decode Entity TODO 需要check一下errno + // 3. decode Entity this.entity.decode(buf); // 4. decode affected rows @@ -96,93 +96,93 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override public long getPayloadContentSize() { - return 1 + header.getPayloadSize() + entity.getPayloadSize() - + Serialization.getNeedBytes(affectedRows); + return header.getPayloadSize() + entity.getPayloadSize() + + Serialization.getNeedBytes(affectedRows) + 1; // operation type } - /** + /* * Get header. */ public ObTableResult getHeader() { return header; } - /** + /* * Set header. */ public void setHeader(ObTableResult header) { this.header = header; } - /** + /* * Get operation type. */ public ObTableOperationType getOperationType() { return operationType; } - /** + /* * Set operation type. */ public void setOperationType(ObTableOperationType operationType) { this.operationType = operationType; } - /** + /* * Get entity. */ public ObITableEntity getEntity() { return entity; } - /** + /* * Set entity. */ public void setEntity(ObITableEntity entity) { this.entity = entity; } - /** + /* * Get affected rows. */ public long getAffectedRows() { return affectedRows; } - /** + /* * Set affected rows. */ public void setAffectedRows(long affectedRows) { this.affectedRows = affectedRows; } - /** + /* * Get execute host. */ public String getExecuteHost() { return executeHost; } - /** + /* * Set execute host. */ public void setExecuteHost(String executeHost) { this.executeHost = executeHost; } - /** + /* * Get execute port. */ public int getExecutePort() { return executePort; } - /** + /* * Set execute port. */ public void setExecutePort(int executePort) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java index 082e5201..8c69844b 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationType.java @@ -17,8 +17,7 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; -import java.util.HashMap; -import java.util.Map; +import java.util.*; public enum ObTableOperationType { @@ -26,10 +25,29 @@ public enum ObTableOperationType { INSERT_OR_UPDATE(4), // INSERT or UPDATE, columns not in arguments will remain unchanged REPLACE(5), // DELETE & INSERT, columns not in arguments will change to default value INCREMENT(6), // the column must be can be cast to long. if exist increase, else insert - APPEND(7);// append column value + APPEND(7), // append column value + SCAN(8), // query + TTL(9), // observer internal type, not used by client + CHECK_AND_INSERT_UP(10), PUT(11), // override row + TRIGGER(12), // internal op type + INVALID(15); private int value; - private static Map map = new HashMap(); + private static Map map = new HashMap(); + private static final List needEncodeQuery = Arrays.asList(false, // GET + false, // INSERT + false, // DEL + false, // UPDATE + false, // INSERT_OR_UPDATE + false, // REPLACE + false, // INCREMENT + false, // APPEND + false, // SCAN + false, // TTL + true, // CHECK_AND_INSERT_UP + false, // PUT + false // INVALID + ); ObTableOperationType(int value) { this.value = value; @@ -41,31 +59,35 @@ public enum ObTableOperationType { } } - /** + /* * Value of. */ public static ObTableOperationType valueOf(int value) { return map.get(value); } - /** + /* * Get value. */ public int getValue() { return value; } - /** + /* * Get byte value. */ public byte getByteValue() { return (byte) value; } - /** + /* * Is readonly. */ public boolean isReadonly() { return this.value == GET.value; } + + public static boolean needEncodeQuery(ObTableOperationType type) { + return needEncodeQuery.get(type.value); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java new file mode 100644 index 00000000..0b02e4cc --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOptionFlag.java @@ -0,0 +1,104 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import java.util.HashMap; +import java.util.Map; + +public enum ObTableOptionFlag { + + DEFAULT(0), RETURNING_ROWKEY(1 << 0), USE_PUT(1 << 1), RETURN_ONE_RES(1 << 2); + + private int value; + private static Map map = new HashMap(); + + ObTableOptionFlag(int value) { + this.value = value; + } + + static { + for (ObTableOptionFlag type : ObTableOptionFlag.values()) { + map.put(type.value, type); + } + } + + /* + * Value of. + */ + public static ObTableOptionFlag valueOf(int value) { + return map.get(value); + } + + /* + * Get value. + */ + public int getValue() { + return value; + } + + /* + * Get byte value. + */ + public byte getByteValue() { + return (byte) value; + } + + /* + * Set isReturningRowKey. + */ + public void setReturningRowkey(boolean returningRowKey) { + if (returningRowKey) { + this.value |= RETURNING_ROWKEY.value; + } + } + + /* + * Get isReturningRowKey. + */ + public boolean isReturningRowKey() { + return this.value == RETURNING_ROWKEY.value; + } + + /* + * Get usePut. + */ + public boolean isUsePut() { + return this.value == USE_PUT.value; + } + + /* + * Set usePut. + */ + public void setUsePut(boolean usePut) { + if (usePut) { + this.value |= USE_PUT.value; + } + } + + public boolean isReturnOneResult() { + return (this.value & RETURN_ONE_RES.value) == 1; + } + + public void setReturnOneResult(boolean returnOneResult) { + if (returnOneResult) { + this.value |= RETURN_ONE_RES.value; + } else { + this.value &= ~(RETURN_ONE_RES.value); + } + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableQueryAndMutateFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableQueryAndMutateFlag.java new file mode 100644 index 00000000..aed1d3ce --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableQueryAndMutateFlag.java @@ -0,0 +1,52 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +public class ObTableQueryAndMutateFlag { + private static final int FLAG_IS_CHECK_AND_EXECUTE = 1 << 0; + private static final int FLAG_IS_CHECK_NOT_EXISTS = 1 << 1; + private long flags = 0; + + public void setIsCheckAndExecute(boolean isCheckAndExecute) { + if (isCheckAndExecute) { + flags |= FLAG_IS_CHECK_AND_EXECUTE; + } else { + flags &= ~FLAG_IS_CHECK_AND_EXECUTE; + } + } + + public void setIsCheckNotExists(boolean isCheckNotExists) { + if (isCheckNotExists) { + flags |= FLAG_IS_CHECK_NOT_EXISTS; + } else { + flags &= ~FLAG_IS_CHECK_NOT_EXISTS; + } + } + + public long getValue() { + return flags; + } + + public boolean isCheckNotExists() { + return (flags & FLAG_IS_CHECK_NOT_EXISTS) != 0; + } + + public boolean isChekAndExecute() { + return (flags & FLAG_IS_CHECK_AND_EXECUTE) != 0; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableResult.java index d7bdb27e..fd65ad95 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableResult.java @@ -28,7 +28,7 @@ public class ObTableResult extends AbstractPayload { private byte[] sqlState; private byte[] msg; - /** + /* * Encode. */ @Override @@ -55,7 +55,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -71,7 +71,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -81,42 +81,52 @@ public long getPayloadContentSize() { + Serialization.getNeedBytes(msg); } - /** + /* * Get errno. */ public int getErrno() { return errno; } - /** + /* * Set errno. */ public void setErrno(int errno) { this.errno = errno; } - /** + /* * Get sql state. */ public byte[] getSqlState() { return sqlState; } - /** + /* * Set sql state. */ public void setSqlState(byte[] sqlState) { this.sqlState = sqlState; } - /** + /* * Get msg. */ public byte[] getMsg() { return msg; } - /** + /* + * Get error msg. + */ + public String getErrMsg() { + if (msg != null && msg.length > 0) { + return new String(msg, 0, msg.length - 1); + } + return new String(msg); + } + + /* * Set msg. */ public void setMsg(byte[] msg) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOp.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOp.java new file mode 100644 index 00000000..79271e00 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOp.java @@ -0,0 +1,170 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +public class ObTableSingleOp extends AbstractPayload { + private ObTableOperationType singleOpType; + private ObTableSingleOpFlag singleOpFlag = new ObTableSingleOpFlag(); + private ObTableSingleOpQuery query = new ObTableSingleOpQuery(); + private List entities = new ArrayList<>(); + + /* + * Encode. + */ + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode header + idx = encodeHeader(bytes, idx); + + // 1. encode op type + byte opTypeVal = singleOpType.getByteValue(); + System.arraycopy(Serialization.encodeI8(opTypeVal), 0, bytes, idx, 1); + idx += 1; + + // 2. encode op flag + long flag = singleOpFlag.getValue(); + int len = Serialization.getNeedBytes(flag); + System.arraycopy(Serialization.encodeVi64(flag), 0, bytes, idx, len); + idx += len; + + // 3. encode single op query + if (ObTableOperationType.needEncodeQuery(singleOpType)) { + len = (int) query.getPayloadSize(); + System.arraycopy(query.encode(), 0, bytes, idx, len); + idx += len; + } + + // 4. encode entities + len = Serialization.getNeedBytes(entities.size()); + System.arraycopy(Serialization.encodeVi64(entities.size()), 0, bytes, idx, len); + idx += len; + for (ObTableSingleOpEntity entity : entities) { + len = (int) entity.getPayloadSize(); + System.arraycopy(entity.encode(), 0, bytes, idx, len); + idx += len; + } + + return bytes; + } + + /* + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + super.decode(buf); + + this.singleOpType = ObTableOperationType.valueOf(Serialization.decodeI8(buf.readByte())); + this.singleOpFlag.setValue(Serialization.decodeVi64(buf)); + if (ObTableOperationType.needEncodeQuery(this.singleOpType)) { + this.query.decode(buf); + } + int len = (int) Serialization.decodeVi64(buf); + for (int i = 0; i < len; i++) { + ObTableSingleOpEntity entity = new ObTableSingleOpEntity(); + entity.decode(buf); + entities.add(entity); + } + + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + long payloadContentSize = Serialization.getNeedBytes(singleOpType.getByteValue()); + payloadContentSize += Serialization.getNeedBytes(singleOpFlag.getValue()); + if (ObTableOperationType.needEncodeQuery(singleOpType)) { + payloadContentSize += query.getPayloadSize(); + } + payloadContentSize += Serialization.getNeedBytes(entities.size()); + for (ObTableSingleOpEntity entity : entities) { + payloadContentSize += entity.getPayloadSize(); + } + return payloadContentSize; + } + + public List getScanRange() { + return query.getScanRanges(); + } + + public void addScanRange(ObNewRange range) + { + this.addScanRange(range); + } + + public void setIsCheckNoExists(boolean isCheckNoExists) { + singleOpFlag.setIsCheckNotExists(isCheckNoExists); + } + + public ObTableOperationType getSingleOpType() { + return singleOpType; + } + + public void setSingleOpType(ObTableOperationType singleOpType) { + this.singleOpType = singleOpType; + } + + public ObTableSingleOpQuery getQuery() { + return query; + } + + public void setQuery(ObTableSingleOpQuery query) { + this.query = query; + } + + public List getEntities() { + return entities; + } + + public void setEntities(List entities) { + this.entities = entities; + } + + public void addEntity(ObTableSingleOpEntity entity) { + this.entities.add(entity); + } + + public List getRowkeyObjs() { + List rowkeyObjs; + if (singleOpType == ObTableOperationType.SCAN) { + throw new IllegalArgumentException("can not get rowkey from scan operation"); + } else if (singleOpType == ObTableOperationType.CHECK_AND_INSERT_UP) { + rowkeyObjs = getScanRange().get(0).getStartKey().getObjs(); + } else { + rowkeyObjs = entities.get(0).getRowkey(); + } + return rowkeyObjs; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpEntity.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpEntity.java new file mode 100644 index 00000000..d9a440f4 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpEntity.java @@ -0,0 +1,369 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjMeta; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObTableSerialUtil; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import java.util.*; + +public class ObTableSingleOpEntity extends AbstractPayload { + private List rowKeyNames = new ArrayList<>(); + private byte[] rowKeyBitMap = null; + private long rowKeyBitLen = 0; + private List aggRowKeyNames = new ArrayList<>(); + private List rowkey = new ArrayList<>(); + + private List propertiesNames = new ArrayList<>(); + private byte[] propertiesBitMap = null; + private long propertiesBitLen = 0; + private List aggPropertiesNames = new ArrayList<>(); + private List propertiesValues = new ArrayList<>(); + + private boolean ignoreEncodePropertiesColumnNames = false; + + public ObTableSingleOpEntity() {} + + /* + * Encode. + */ + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode header + idx = encodeHeader(bytes, idx); + + // 1. encode rowKey bitmap + int len = Serialization.getNeedBytes(rowKeyBitLen); + System.arraycopy(Serialization.encodeVi64(rowKeyBitLen), 0, bytes, idx, len); + idx += len; + for (byte b : rowKeyBitMap) { + System.arraycopy(Serialization.encodeI8(b), 0, bytes, idx, 1); + idx += 1; + } + + // 2. encode rowkey + len = Serialization.getNeedBytes(rowkey.size()); + System.arraycopy(Serialization.encodeVi64(rowkey.size()), 0, bytes, idx, len); + idx += len; + for (ObObj obj : rowkey) { + len = ObTableSerialUtil.getEncodedSize(obj); + System.arraycopy(ObTableSerialUtil.encode(obj), 0, bytes, idx, len); + idx += len; + } + + // 3. encode property bitmap + if (ignoreEncodePropertiesColumnNames) { + len = Serialization.getNeedBytes(0L); + System.arraycopy(Serialization.encodeVi64(0L), 0, bytes, idx, len); + idx += len; + } else { + len = Serialization.getNeedBytes(propertiesBitLen); + System.arraycopy(Serialization.encodeVi64(propertiesBitLen), 0, bytes, idx, len); + idx += len; + for (byte b : propertiesBitMap) { + System.arraycopy(Serialization.encodeI8(b), 0, bytes, idx, 1); + idx += 1; + } + } + + // 4. encode properties values + len = Serialization.getNeedBytes(propertiesValues.size()); + System.arraycopy(Serialization.encodeVi64(propertiesValues.size()), 0, bytes, idx, len); + idx += len; + for (ObObj obj : propertiesValues) { + len = ObTableSerialUtil.getEncodedSize(obj); + System.arraycopy(ObTableSerialUtil.encode(obj), 0, bytes, idx, len); + idx += len; + } + + return bytes; + } + + private byte[] parseBitMap(long bitLen, List aggColumnNames, List columnNames, ByteBuf buf) { + byte[] bitMap = new byte[(int) Math.ceil(bitLen / 8.0)]; + if (bitLen == 0) { + // is same properties names + columnNames.addAll(aggColumnNames); + } else { + for (int i = 0; i < bitMap.length; i++) { + bitMap[i] = Serialization.decodeI8(buf); + for (int j = 0; j < 8; j++) { + if ((bitMap[i] & (1 << j)) != 0) { + if (i * 8 + j < aggColumnNames.size()) { + columnNames.add(aggColumnNames.get(i * 8 + j)); + } + } + } + } + } + return bitMap; + } + + /* + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + // 0. decode header + super.decode(buf); + + // 1. rowkey bitmap + rowKeyBitLen = Serialization.decodeVi64(buf); + rowKeyBitMap = parseBitMap(rowKeyBitLen, aggRowKeyNames, rowKeyNames, buf); + + // 2. rowkey obobj + int len = (int) Serialization.decodeVi64(buf); + for (int i = 0; i < len; i++) { + ObObj obj = new ObObj(); + ObTableSerialUtil.decode(buf, obj); + rowkey.add(obj); + } + + // 3. properties bitmap + propertiesBitLen = Serialization.decodeVi64(buf); + propertiesBitMap = parseBitMap(propertiesBitLen, aggPropertiesNames, propertiesNames, buf); + + // 4. properties obobj + len = (int) Serialization.decodeVi64(buf); + for (int i = 0; i < len; i++) { + ObObj obj = new ObObj(); + ObTableSerialUtil.decode(buf, obj); + propertiesValues.add(obj); + } + + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + long payloadContentSize = 0; + + payloadContentSize += Serialization.getNeedBytes(rowKeyBitLen); + payloadContentSize += rowKeyBitMap.length; + + payloadContentSize += Serialization.getNeedBytes(rowkey.size()); + for (ObObj obj : rowkey) { + payloadContentSize += ObTableSerialUtil.getEncodedSize(obj); + } + + if (ignoreEncodePropertiesColumnNames) { + payloadContentSize += Serialization.getNeedBytes(0L); + } else { + payloadContentSize += Serialization.getNeedBytes(propertiesBitLen); + payloadContentSize += propertiesBitMap.length; + } + + payloadContentSize += Serialization.getNeedBytes(propertiesValues.size()); + for (ObObj obj : propertiesValues) { + payloadContentSize += ObTableSerialUtil.getEncodedSize(obj); + } + + return payloadContentSize; + } + + public static boolean areArraysSameLengthOrBothNull(Object[] a, Object[] b) { + if (a == null && b == null) { + return true; + } else if (a != null && b != null && a.length == b.length) { + return true; + } else { + return false; + } + } + + public static ObTableSingleOpEntity getInstance(String[] rowKeyNames, Object[] rowKey, + String[] propertiesNames, Object[] propertiesValues) { + ObTableSingleOpEntity entity = new ObTableSingleOpEntity(); + if (!areArraysSameLengthOrBothNull(rowKeyNames, rowKey)) { + throw new IllegalArgumentException(String.format( + "rowKey names length should be equals to rowKey values length, rowkeyNames: %s, rowkey: %s,", + rowKeyNames, rowKey)); + } + + if (rowKey != null) { + for (int i = 0; i < rowKey.length; i++) { + String name = rowKeyNames[i]; + Object rowkey = rowKey[i]; + ObObj obj = ObObj.getInstance(rowkey); + entity.addRowKeyValue(name, obj); + } + } + if (propertiesNames != null) { + for (int i = 0; i < propertiesNames.length; i++) { + String name = propertiesNames[i]; + Object value = null; + if (propertiesValues != null) { + value = propertiesValues[i]; + } + ObObj c = ObObj.getInstance(value); + entity.addPropertyValue(name, c); + } + } + return entity; + } + + // Support class, which is used for column name sorted + private static class ColumnNamePair implements Comparable { + long number; + // we could use idx here, and adjust obj after compare + ObObj obj; + + ColumnNamePair(long number, ObObj obj) { + this.number = number; + this.obj = obj; + } + + @Override + public int compareTo(ColumnNamePair other) { + return Long.compare(this.number, other.number); + } + } + + /* + * isSamePropertiesColumnNamesLen check whether length is equal + */ + public boolean isSamePropertiesColumnNamesLen(int columnNameIdxMapLen) { + return columnNameIdxMapLen == this.propertiesNames.size(); + } + + public void adjustRowkeyColumnName(Map columnNameIdxMap) { + this.rowKeyBitLen = columnNameIdxMap.size(); + int size = (int) Math.ceil(columnNameIdxMap.size() / 8.0); + byte[] byteArray = new byte[size]; + List columnNameIdx = new LinkedList<>(); + + for (String name : rowKeyNames) { + Long index = columnNameIdxMap.get(name); + columnNameIdx.add(index); + if (index != null) { + int byteIndex = index.intValue() / 8; + int bitIndex = index.intValue() % 8; + byteArray[byteIndex] |= (byte) (1 << bitIndex); + } + } + + List pairs = new ArrayList<>(); + for (int i = 0; i < columnNameIdx.size(); i++) { + pairs.add(new ColumnNamePair(columnNameIdx.get(i), rowkey.get(i))); + } + + Collections.sort(pairs); + + rowkey = new ArrayList<>(pairs.size()); + for (ColumnNamePair pair : pairs) { + rowkey.add(pair.obj); + } + + this.rowKeyBitMap = byteArray; + } + + public void adjustPropertiesColumnName(Map columnNameIdxMap) { + if (!ignoreEncodePropertiesColumnNames) { + this.propertiesBitLen = columnNameIdxMap.size(); + } + int size = (int) Math.ceil(columnNameIdxMap.size() / 8.0); + byte[] byteArray = new byte[size]; + List columnNameIdx = new LinkedList<>(); + + for (String name : propertiesNames) { + Long index = columnNameIdxMap.get(name); + columnNameIdx.add(index); + if (index != null) { + int byteIndex = index.intValue() / 8; + int bitIndex = index.intValue() % 8; + byteArray[byteIndex] |= (byte) (1 << bitIndex); + } + } + + List pairs = new ArrayList<>(); + for (int i = 0; i < columnNameIdx.size(); i++) { + pairs.add(new ColumnNamePair(columnNameIdx.get(i), propertiesValues.get(i))); + } + + Collections.sort(pairs); + + propertiesValues = new ArrayList<>(pairs.size()); + for (ColumnNamePair pair : pairs) { + propertiesValues.add(pair.obj); + } + + this.propertiesBitMap = byteArray; + } + + public void addRowKeyValue(String rowKeyName, ObObj rowKeyValue) { + this.rowKeyNames.add(rowKeyName); + this.rowkey.add(rowKeyValue); + } + + public void addPropertyValue(String propertyName, ObObj propertyValue) { + this.propertiesNames.add(propertyName); + this.propertiesValues.add(propertyValue); + } + + public List getRowKeyNames() { + return this.rowKeyNames; + } + + public List getPropertiesNames() { + return this.propertiesNames; + } + + public void setIgnoreEncodePropertiesColumnNames(boolean ignoreEncodePropertiesColumnNames) { + this.ignoreEncodePropertiesColumnNames = ignoreEncodePropertiesColumnNames; + } + + public boolean ignoreEncodePropertiesColumnNames() { + return this.ignoreEncodePropertiesColumnNames; + } + + public void setAggRowKeyNames(List columnNames) { + this.aggRowKeyNames = columnNames; + } + + public void setAggPropertiesNames(List columnNames) { + this.aggPropertiesNames = columnNames; + } + + public Map getSimpleProperties() { + Map values = new HashMap((int) propertiesValues.size()); + for (int i = 0; i < propertiesValues.size(); i++) { + values.put(propertiesNames.get(i), propertiesValues.get(i).getValue()); + } + return values; + } + + public List getRowkey() { + return rowkey; + } + + public void setRowkey(List rowkey) { + this.rowkey = rowkey; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpFlag.java new file mode 100644 index 00000000..30dbff3b --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpFlag.java @@ -0,0 +1,43 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +public class ObTableSingleOpFlag { + private static final int FLAG_IS_CHECK_NOT_EXISTS = 1 << 0; + private long flags = 0; + + public void setIsCheckNotExists(boolean isCheckNotExists) { + if (isCheckNotExists) { + flags |= FLAG_IS_CHECK_NOT_EXISTS; + } else { + flags &= ~FLAG_IS_CHECK_NOT_EXISTS; + } + } + + public long getValue() { + return flags; + } + + public boolean isCheckNotExists() { + return (flags & FLAG_IS_CHECK_NOT_EXISTS) != 0; + } + + void setValue(long value) { + flags = value; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpQuery.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpQuery.java new file mode 100644 index 00000000..7cae5744 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpQuery.java @@ -0,0 +1,232 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObTableSerialUtil; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import java.util.*; + +public class ObTableSingleOpQuery extends AbstractPayload { + private String indexName; + private List scanRangeColumns = new ArrayList<>(); + private byte[] scanRangeBitMap = null; + private long scanRangeBitLen = 0; + private List aggColumnNames = new ArrayList<>(); + + private List scanRanges = new ArrayList<>(); + + private String filterString; + + /* + * Encode. + */ + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode header + idx = encodeHeader(bytes, idx); + + // 1. encode index name + int len = Serialization.getNeedBytes(indexName); + System.arraycopy(Serialization.encodeVString(indexName), 0, bytes, idx, len); + idx += len; + + // 2. encode scan ranges columns + len = Serialization.getNeedBytes(scanRangeBitLen); + System.arraycopy(Serialization.encodeVi64(scanRangeBitLen), 0, bytes, idx, len); + idx += len; + for (byte b : scanRangeBitMap) { + System.arraycopy(Serialization.encodeI8(b), 0, bytes, idx, 1); + idx += 1; + } + + // 3. encode scan ranges + len = Serialization.getNeedBytes(scanRanges.size()); + System.arraycopy(Serialization.encodeVi64(scanRanges.size()), 0, bytes, idx, len); + idx += len; + for (ObNewRange range : scanRanges) { + len = ObTableSerialUtil.getEncodedSize(range); + System.arraycopy(ObTableSerialUtil.encode(range), 0, bytes, idx, len); + idx += len; + } + + // 4. encode filter string + len = Serialization.getNeedBytes(filterString); + System.arraycopy(Serialization.encodeVString(filterString), 0, bytes, idx, len); + idx += len; + + return bytes; + } + + /* + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + // 0. decode header + super.decode(buf); + + // 1. decode tablet id + this.indexName = Serialization.decodeVString(buf); + + // 2. decode scan ranges columns + scanRangeBitLen = Serialization.decodeVi64(buf); + scanRangeBitMap = new byte[(int) Math.ceil(scanRangeBitLen / 8.0)]; + for (int i = 0; i < scanRangeBitMap.length; i++) { + scanRangeBitMap[i] = Serialization.decodeI8(buf); + for (int j = 0; j < 8; i++) { + if ((scanRangeBitMap[i] & (1 << j)) != 0) { + if (i * 8 + j < aggColumnNames.size()) { + scanRangeColumns.add(aggColumnNames.get(i * 8 + j)); + } + } + } + } + + // 3. decode scan ranges + int len = (int) Serialization.decodeVi64(buf); + for (int i = 0; i < len; i++) { + ObNewRange range = new ObNewRange(); + ObTableSerialUtil.decode(buf, range); + scanRanges.add(range); + } + + // 4. decode filter string + this.filterString = Serialization.decodeVString(buf); + + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + long payloadContentSize = 0; + + payloadContentSize += Serialization.getNeedBytes(scanRangeBitLen); + payloadContentSize += scanRangeBitMap.length; + + payloadContentSize += Serialization.getNeedBytes(scanRanges.size()); + for (ObNewRange range : scanRanges) { + payloadContentSize += ObTableSerialUtil.getEncodedSize(range); + } + + return payloadContentSize + Serialization.getNeedBytes(indexName) + + Serialization.getNeedBytes(filterString); + } + + // Support class, which is used for column name sorted + private static class ColumnNamePair implements Comparable { + long number; + long origin_idx; + + ColumnNamePair(long number, long obj) { + this.number = number; + this.origin_idx = obj; + } + + @Override + public int compareTo(ColumnNamePair other) { + return Long.compare(this.number, other.number); + } + } + + /* + * adjustRowkeyColumnName should be execute in the last + */ + public void adjustScanRangeColumns(Map columnNameIdxMap) { + this.scanRangeBitLen = columnNameIdxMap.size(); + int size = (int) Math.ceil(columnNameIdxMap.size() / 8.0); + byte[] byteArray = new byte[size]; + List columnNameIdx = new LinkedList<>(); + + + for (String name : scanRangeColumns) { + Long index = columnNameIdxMap.get(name); + columnNameIdx.add(index); + if (index != null) { + int byteIndex = index.intValue() / 8; + int bitIndex = index.intValue() % 8; + byteArray[byteIndex] |= (byte) (1 << bitIndex); + } + } + + List pairs = new ArrayList<>(); + for (int i = 0; i < columnNameIdx.size(); i++) { + pairs.add(new ColumnNamePair(columnNameIdx.get(i), i)); + } + + Collections.sort(pairs); + + for (ObNewRange range : scanRanges) { + List startKey= range.getStartKey().getObjs(); + List endKey= range.getStartKey().getObjs(); + List adjustStartKey = new ArrayList<>(startKey.size()); + List adjustEndtKey = new ArrayList<>(endKey.size()); + + for (ColumnNamePair pair : pairs) { + adjustStartKey.add(startKey.get((int) pair.origin_idx)); + adjustEndtKey.add(endKey.get((int) pair.origin_idx)); + } + range.getStartKey().setObjs(adjustStartKey); + range.getEndKey().setObjs(adjustEndtKey); + } + + this.scanRangeBitMap = byteArray; + } + + public List getScanRanges() { + return scanRanges; + } + + public void setScanRanges(List scanRanges) { + this.scanRanges = scanRanges; + } + + public void addScanRange(ObNewRange scanRange) { + this.scanRanges.add(scanRange); + } + + public void addScanRangeColumns(List scanRangeColumns) { + this.scanRangeColumns = scanRangeColumns; + } + + public String getFilterString() { + return filterString; + } + + public void setFilterString(String filterString) { + this.filterString = filterString; + } + + public List getScanRangeColumns() { + return scanRangeColumns; + } + + public void setAggColumnNames(List columnNames) { + this.aggColumnNames = columnNames; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpResult.java new file mode 100644 index 00000000..418af61a --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpResult.java @@ -0,0 +1,200 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import java.util.ArrayList; +import java.util.List; + +public class ObTableSingleOpResult extends AbstractPayload { + private ObTableResult header = new ObTableResult(); + private ObTableOperationType operationType = ObTableOperationType.GET; + private ObTableSingleOpEntity entity = new ObTableSingleOpEntity(); + private long affectedRows = 0; + private String executeHost; + private int executePort; + private List propertiesColumnNames = new ArrayList<>(); + /* + * Get pcode. + */ + @Override + public int getPcode() { + return Pcodes.OB_TABLE_API_LS_EXECUTE; + } + + /* + * Encode. + */ + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode header + int len = (int) Serialization.getObUniVersionHeaderLength(getVersion(), getPayloadSize()); + byte[] header = Serialization.encodeObUniVersionHeader(getVersion(), getPayloadSize()); + System.arraycopy(header, 0, bytes, idx, len); + idx += len; + + // 1. encode ObTableResult + len = (int) this.header.getPayloadSize(); + System.arraycopy(this.header.encode(), 0, bytes, idx, len); + idx += len; + + // 2. encode ObTableOperationResult + System.arraycopy(Serialization.encodeI8(operationType.getByteValue()), 0, bytes, idx, 1); + idx += 1; + + // 3. encode entity + len = (int) entity.getPayloadSize(); + System.arraycopy(entity.encode(), 0, bytes, idx, len); + idx += len; + + len = Serialization.getNeedBytes(affectedRows); + System.arraycopy(Serialization.encodeVi64(affectedRows), 0, bytes, idx, len); + + return bytes; + } + + /* + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + // 0. decode version + super.decode(buf); + + // 1. decode ObTableResult + this.header.decode(buf); + + // 2. decode types + this.operationType = ObTableOperationType.valueOf(Serialization.decodeI8(buf.readByte())); + + // 3. decode Entity + this.entity.setAggPropertiesNames(propertiesColumnNames); + this.entity.decode(buf); + + // 4. decode affected rows + this.affectedRows = Serialization.decodeVi64(buf); + + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + return header.getPayloadSize() + + entity.getPayloadSize() + + Serialization.getNeedBytes(affectedRows) + + 1; // operation type + } + + /* + * Get header. + */ + public ObTableResult getHeader() { + return header; + } + + /* + * Set header. + */ + public void setHeader(ObTableResult header) { + this.header = header; + } + + /* + * Get operation type. + */ + public ObTableOperationType getOperationType() { + return operationType; + } + + /* + * Set operation type. + */ + public void setOperationType(ObTableOperationType operationType) { + this.operationType = operationType; + } + + /* + * Get entity. + */ + public ObTableSingleOpEntity getEntity() { + return entity; + } + + /* + * Set entity. + */ + public void setEntity(ObTableSingleOpEntity entity) { + this.entity = entity; + } + + /* + * Get affected rows. + */ + public long getAffectedRows() { + return affectedRows; + } + + /* + * Set affected rows. + */ + public void setAffectedRows(long affectedRows) { + this.affectedRows = affectedRows; + } + + /* + * Get execute host. + */ + public String getExecuteHost() { + return executeHost; + } + + /* + * Set execute host. + */ + public void setExecuteHost(String executeHost) { + this.executeHost = executeHost; + } + + /* + * Get execute port. + */ + public int getExecutePort() { + return executePort; + } + + /* + * Set execute port. + */ + public void setExecutePort(int executePort) { + this.executePort = executePort; + } + + public void setPropertiesColumnNames(List columnNames) { + this.propertiesColumnNames = columnNames; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpType.java new file mode 100644 index 00000000..8f06047c --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableSingleOpType.java @@ -0,0 +1,74 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.Constants; + +import java.util.HashMap; +import java.util.Map; + +public enum ObTableSingleOpType { + SINGLE_GET(0), + SINGLE_INSERT(1), + SINGLE_DEL(2), + SINGLE_UPDATE(3), + SINGLE_INSERT_OR_UPDATE(4), + SINGLE_REPLACE(5), + SINGLE_INCREMENT(6), + SINGLE_APPEND(7), + SINGLE_MAX(63), // reserved + SYNC_QUERY(64), + ASYNC_QUERY(65), + QUERY_AND_MUTATE(66), + SINGLE_OP_TYPE_MAX(67); + + private int value; + private static Map map = new HashMap<>(); + + ObTableSingleOpType(int value) { + this.value = value; + } + + static { + for (ObTableSingleOpType type : ObTableSingleOpType.values()) { + map.put(type.value, type); + } + } + + /* + * Value of. + */ + public static ObTableSingleOpType valueOf(byte value) { + return map.get(value); + } + + /* + * Get value. + */ + public byte getValue() { + return (byte) value; + } + + /* + * Set value. + */ + public void setValue(byte value) { + this.value = value; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableStreamRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableStreamRequest.java index 82c715d8..ebdc0449 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableStreamRequest.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableStreamRequest.java @@ -30,7 +30,7 @@ public class ObTableStreamRequest extends AbstractPayload { private long sessionId; private short flag = 0x7; // let ObServer determine the ob log level. - /** + /* * Get pcode. */ @Override @@ -38,7 +38,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_EXECUTE_QUERY; } - /** + /* * Encode. */ @Override @@ -52,7 +52,7 @@ public byte[] encode() { return bytes; } - /** + /* * Get payload content size. */ @Override @@ -60,7 +60,7 @@ public long getPayloadContentSize() { return 0; } - /** + /* * Set stream next. */ public void setStreamNext() { @@ -68,7 +68,7 @@ public void setStreamNext() { flag |= STREAM_FLAG; } - /** + /* * Set stream last. */ public void setStreamLast() { @@ -76,28 +76,28 @@ public void setStreamLast() { flag |= STREAM_FLAG; } - /** + /* * Set timeout. */ public void setTimeout(long timeout) { this.timeout = timeout; } - /** + /* * Get session id. */ public long getSessionId() { return sessionId; } - /** + /* * Set session id. */ public void setSessionId(long sessionId) { this.sessionId = sessionId; } - /** + /* * Get flag. */ public short getFlag() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOp.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOp.java new file mode 100644 index 00000000..74359104 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOp.java @@ -0,0 +1,193 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Constants; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import java.util.*; + +/* +OB_UNIS_DEF_SERIALIZE(ObTableTabletOp, + table_id_, + tablet_id_, + option_flag_, + single_ops_); + */ +public class ObTableTabletOp extends AbstractPayload { + private List singleOperations = new ArrayList<>(); + private long tabletId = Constants.INVALID_TABLET_ID; // i64 + + private Set rowKeyNamesSet = new LinkedHashSet<>(); + private Set propertiesNamesSet = new LinkedHashSet<>(); + ObTableTabletOpFlag optionFlag = new ObTableTabletOpFlag(); + + private static final int tabletIdSize = 8; + + /* + * Encode. + */ + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode header + idx = encodeHeader(bytes, idx); + + // 1. encode tablet id + System.arraycopy(Serialization.encodeI64(tabletId), 0, bytes, idx, 8); + idx += 8; + + // 2. encode option flag + int len = Serialization.getNeedBytes(optionFlag.getValue()); + System.arraycopy(Serialization.encodeVi64(optionFlag.getValue()), 0, bytes, idx, len); + idx += len; + + // 4. encode Operation + len = Serialization.getNeedBytes(singleOperations.size()); + System.arraycopy(Serialization.encodeVi64(singleOperations.size()), 0, bytes, idx, len); + idx += len; + for (ObTableSingleOp singleOperation : singleOperations) { + len = (int) singleOperation.getPayloadSize(); + System.arraycopy(singleOperation.encode(), 0, bytes, idx, len); + idx += len; + } + + return bytes; + } + + /* + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + // 0. decode header + super.decode(buf); + + // 1. decode tablet id + this.tabletId = Serialization.decodeI64(buf); + + // 2. decode other flags + long flagsValue = Serialization.decodeVi64(buf); + optionFlag.setValue(flagsValue); + + // 3. decode Operation + int len = (int) Serialization.decodeVi64(buf); + for (int i = 0; i < len; i++) { + ObTableSingleOp singleOperation = new ObTableSingleOp(); + singleOperation.decode(buf); + singleOperations.add(singleOperation); + } + + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + long payloadContentSize = 0; + payloadContentSize += Serialization.getNeedBytes(singleOperations.size()); + for (ObTableSingleOp operation : singleOperations) { + payloadContentSize += operation.getPayloadSize(); + } + + return payloadContentSize + tabletIdSize + Serialization.getNeedBytes(optionFlag.getValue()); + } + + /* + * Get table operations. + */ + public List getSingleOperations() { + return singleOperations; + } + + /* + * Set table operations. + */ + public void setSingleOperations(List singleOperations) { + setIsSameType(true); + setIsSamePropertiesNames(true); + ObTableOperationType prevType = null; + List prevPropertiesNames = null; + for (ObTableSingleOp o : singleOperations) { + // get union column names + rowKeyNamesSet.addAll(o.getQuery().getScanRangeColumns()); + for (ObTableSingleOpEntity e: o.getEntities()) { + rowKeyNamesSet.addAll(e.getRowKeyNames()); + propertiesNamesSet.addAll(e.getPropertiesNames()); + + if (!isSamePropertiesNames()) { + // do nothing + } else if (prevPropertiesNames != null && isSamePropertiesNames() && + !prevPropertiesNames.equals(e.getPropertiesNames())) { + setIsSamePropertiesNames(false); + } else { + prevPropertiesNames = e.getPropertiesNames(); + } + } + + if (!isSameType()) { + // do nothing + } else if (prevType != null && prevType != o.getSingleOpType()) { + setIsSameType(false); + } else { + prevType = o.getSingleOpType(); + } + } + + if (isSameType() && singleOperations.get(0).getSingleOpType() == ObTableOperationType.GET) { + setIsReadOnly(true); + } + this.singleOperations = singleOperations; + } + + public void setTabletId(long tabletId) { + this.tabletId = tabletId; + } + + public long getTabletId() { + return tabletId; + } + + public boolean isSameType() { return optionFlag.getFlagIsSameType(); } + + public boolean isSamePropertiesNames() { + return optionFlag.getFlagIsSamePropertiesNames(); + } + + public void setIsSameType(boolean isSameType) { optionFlag.setFlagIsSameType(isSameType);} + + public void setIsReadOnly(boolean isReadOnly) { optionFlag.setFlagIsReadOnly(isReadOnly);} + + public void setIsSamePropertiesNames(boolean isSamePropertiesNames) { + optionFlag.setFlagIsSamePropertiesNames(isSamePropertiesNames); + } + + public Set getRowKeyNamesSet() { + return rowKeyNamesSet; + } + + public Set getPropertiesNamesSet() { + return propertiesNamesSet; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpFlag.java new file mode 100644 index 00000000..0e28ea59 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpFlag.java @@ -0,0 +1,70 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +public class ObTableTabletOpFlag { + private static final int FLAG_IS_SAME_TYPE = 1 << 0; + // Maybe useless, we use isSameProperties flag from LSOp + private static final int FLAG_IS_SAME_PROPERTIES_NAMES = 1 << 1; + private static final int FLAG_IS_READ_ONLY = 1 << 2; + private long flags = 0; + + public void setFlagIsSameType(boolean isSameType) { + if (isSameType) { + flags |= FLAG_IS_SAME_TYPE; + } else { + flags &= ~FLAG_IS_SAME_TYPE; + } + } + + public void setFlagIsSamePropertiesNames(boolean isSamePropertiesNames) { + if (isSamePropertiesNames) { + flags |= FLAG_IS_SAME_PROPERTIES_NAMES; + } else { + flags &= ~FLAG_IS_SAME_PROPERTIES_NAMES; + } + } + + public void setFlagIsReadOnly(boolean isReadOnly) { + if (isReadOnly) { + flags |= FLAG_IS_READ_ONLY; + } else { + flags &= ~FLAG_IS_READ_ONLY; + } + } + + public long getValue() { + return flags; + } + + public void setValue(long flags) { + this.flags = flags; + } + + public boolean getFlagIsSameType() { + return (flags & FLAG_IS_SAME_TYPE) != 0; + } + + public boolean getFlagIsSamePropertiesNames() { + return (flags & FLAG_IS_SAME_PROPERTIES_NAMES) != 0; + } + + public boolean getFlagIsReadOnly() { + return (flags & FLAG_IS_READ_ONLY) != 0; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpResult.java new file mode 100644 index 00000000..4585a3b0 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableTabletOpResult.java @@ -0,0 +1,130 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +import java.util.ArrayList; +import java.util.List; + +import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; +import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; + +public class ObTableTabletOpResult extends AbstractPayload { + private List results = new ArrayList(); + private List propertiesColumnNames = new ArrayList<>(); + + + @Override + public int getPcode() { + return Pcodes.OB_TABLE_API_LS_EXECUTE; + } + + /* + * Encode. + */ + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode header + // ver + plen + payload + int headerLen = (int) getObUniVersionHeaderLength(getVersion(), getPayloadContentSize()); + System.arraycopy(encodeObUniVersionHeader(getVersion(), getPayloadContentSize()), 0, bytes, + idx, headerLen); + idx += headerLen; + + // 1. encode results + int len = Serialization.getNeedBytes(this.results.size()); + System.arraycopy(Serialization.encodeVi64(this.results.size()), 0, bytes, idx, len); + idx += len; + + for (ObTableSingleOpResult result : results) { + len = (int) result.getPayloadSize(); + System.arraycopy(result.encode(), 0, bytes, idx, len); + idx += len; + } + + return bytes; + } + + /* + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + // 0. decode version + super.decode(buf); + + // 1. decode results + int len = (int) Serialization.decodeVi64(buf); + results = new ArrayList(len); + for (int i = 0; i < len; i++) { + ObTableSingleOpResult obTableSingleOpResult = new ObTableSingleOpResult(); + obTableSingleOpResult.setPropertiesColumnNames(propertiesColumnNames); + obTableSingleOpResult.decode(buf); + results.add(obTableSingleOpResult); + } + + return this; + } + + /* + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + long payloadContentSize = 0; + payloadContentSize += Serialization.getNeedBytes(results.size()); + for (ObTableSingleOpResult result : results) { + payloadContentSize += result.getPayloadSize(); + } + + return payloadContentSize; + } + + /* + * Get results. + */ + public List getResults() { + return results; + } + + /* + * Set results. + */ + public void setResults(List results) { + this.results = results; + } + + /* + * Add result. + */ + public void addResult(ObTableSingleOpResult result) { + this.results.add(result); + } + + public void setPropertiesColumnNames(List propertiesColumnNames) { + this.propertiesColumnNames = propertiesColumnNames; + } +} + diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregation.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregation.java new file mode 100644 index 00000000..8d4983bf --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregation.java @@ -0,0 +1,119 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation; + +import com.alipay.oceanbase.rpc.ObClusterTableQuery; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; + +import java.util.ArrayList; +import java.util.List; + +public class ObTableAggregation { + + //this message is used to record the aggregation order and the corresponding aggregation name + private List message = new ArrayList<>(); + private ObClusterTableQuery tablequery; + + public ObTableAggregation(ObClusterTableQuery obClusterTableQuery) { + this.tablequery = obClusterTableQuery; + } + + public void min(String columnName) { + this.tablequery.addAggregation(ObTableAggregationType.MIN, columnName); + message.add("min(" + columnName + ")"); + } + + public void min(String columnName, String aliasName) { + this.tablequery.addAggregation(ObTableAggregationType.MIN, columnName); + message.add("min(" + aliasName + ")"); + } + + public void max(String columnName) { + this.tablequery.addAggregation(ObTableAggregationType.MAX, columnName); + message.add("max(" + columnName + ")"); + } + + public void max(String columnName, String aliasName) { + this.tablequery.addAggregation(ObTableAggregationType.MAX, columnName); + message.add("max(" + aliasName + ")"); + } + + public void count() { + this.tablequery.addAggregation(ObTableAggregationType.COUNT, "*"); + message.add("count(*)"); + } + public void count(String aliasName) { + this.tablequery.addAggregation(ObTableAggregationType.COUNT, "*"); + message.add("count(" + aliasName + ")"); + } + + public void sum(String columnName) { + this.tablequery.addAggregation(ObTableAggregationType.SUM, columnName); + message.add("sum(" + columnName + ")"); + } + + public void sum(String columnName, String aliasName) { + this.tablequery.addAggregation(ObTableAggregationType.SUM, columnName); + message.add("sum(" + aliasName + ")"); + } + + public void avg(String columnName) { + this.tablequery.addAggregation(ObTableAggregationType.AVG, columnName); + message.add("avg(" + columnName + ")"); + } + + public void avg(String columnName, String aliasName) { + this.tablequery.addAggregation(ObTableAggregationType.AVG, columnName); + message.add("avg(" + aliasName + ")"); + } + + public void setScanRangeColumns(String... columns) { + this.tablequery.setScanRangeColumns(columns); + } + + public void addScanRange(Object start, Object end) { + this.tablequery.addScanRange(start, end); + } + + public void addScanRange(Object[] start, Object[] end) { + this.tablequery.addScanRange(start, end); + } + + public void indexName(String indexName) { + this.tablequery.indexName(indexName); + } + + public void setFilter(ObTableFilter filter) { + this.tablequery.setFilter(filter); + } + + public void limit(int offset, int limit) { this.tablequery.limit(offset, limit); } + + public void setOperationTimeout(long operationTimeout) { + this.tablequery.setOperationTimeout(operationTimeout); + } + + public ObTableAggregationResult execute() throws Exception { + if (this.message.size() == 0) { + throw new IllegalArgumentException("please add aggregations."); + } + // In order to get cache size. + this.tablequery.select(message.toArray(new String[message.size()])); + return new ObTableAggregationResult(this.tablequery.execute()); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationResult.java new file mode 100644 index 00000000..21121afa --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationResult.java @@ -0,0 +1,60 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation; + +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.mutation.Row; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; + +import java.util.HashMap; +import java.util.Map; + +public class ObTableAggregationResult { + + private Map row = new HashMap<>(); + private final QueryResultSet queryResultSet; + + public ObTableAggregationResult(QueryResultSet queryResultSet) throws Exception { + this.queryResultSet = queryResultSet; + this.init(); + } + + /* + * Init for aggregation result. + */ + public void init() throws Exception { + if (this.queryResultSet.next()) { + row = this.queryResultSet.getRow(); + } + // else do nothing. + } + + /* + * Get an aggregation result from the column name. + */ + public Object get(String columName) throws Exception { + return row.get(columName); + } + + /* + * Get the aggregation row. + */ + public Row getRow() throws Exception { + return new Row(this.row); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationSingle.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationSingle.java new file mode 100644 index 00000000..ea70e6b8 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationSingle.java @@ -0,0 +1,63 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.util.Serialization; + +import static com.alipay.oceanbase.rpc.util.Serialization.encodeObUniVersionHeader; +import static com.alipay.oceanbase.rpc.util.Serialization.getObUniVersionHeaderLength; + +public class ObTableAggregationSingle extends AbstractPayload { + + private ObTableAggregationType aggType; + private String aggColumn; + + public ObTableAggregationSingle(ObTableAggregationType aggType, String aggColumn) { + this.aggColumn = aggColumn; + this.aggType = aggType; + } + + /* + * Serialize. + */ + public byte[] encode() { + byte[] bytes = new byte[(int) this.getPayloadSize()]; + int idx = 0; + + int headerLen = (int) getObUniVersionHeaderLength(getVersion(), + this.getPayloadContentSize()); + System.arraycopy(encodeObUniVersionHeader(getVersion(), this.getPayloadContentSize()), 0, + bytes, idx, headerLen); + idx += headerLen; + + int len = Serialization.getNeedBytes(aggType.getByteValue()); + System.arraycopy(Serialization.encodeI8(aggType.getByteValue()), 0, bytes, idx, len); + idx += len; + + len = Serialization.getNeedBytes(aggColumn); + System.arraycopy(Serialization.encodeVString(aggColumn), 0, bytes, idx, len); + + return bytes; + } + + public long getPayloadContentSize() { + return Serialization.getNeedBytes(aggType.getByteValue()) + + Serialization.getNeedBytes(aggColumn); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationType.java new file mode 100644 index 00000000..58e55e7a --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/aggregation/ObTableAggregationType.java @@ -0,0 +1,57 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation; + +public enum ObTableAggregationType { + INVAILD(0), MAX(1), MIN(2), COUNT(3), SUM(4), AVG(5); + + private int value; + + /* + * Get agg type from byte + */ + public static ObTableAggregationType fromByte(byte i) { + switch (i) { + case 0: + return INVAILD; + case 1: + return MAX; + case 2: + return MIN; + case 3: + return COUNT; + case 4: + return SUM; + case 5: + return AVG; + default: + throw new IllegalArgumentException("Invalid value: " + i); + } + } + + /* + * For serialize. + */ + public byte getByteValue() { + return (byte) value; + } + + ObTableAggregationType(int value) { + this.value = value; + } +}; diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutate.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutate.java index 91c58da5..396f1814 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutate.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutate.java @@ -20,6 +20,7 @@ import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableBatchOperation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableQueryAndMutateFlag; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQuery; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; @@ -36,11 +37,12 @@ */ public class ObTableQueryAndMutate extends AbstractPayload { - private ObTableQuery tableQuery; - private ObTableBatchOperation mutations; - private boolean returnAffectedEntity = true; + private ObTableQuery tableQuery; + private ObTableBatchOperation mutations; + private boolean returnAffectedEntity = false; + private ObTableQueryAndMutateFlag queryAndMutateFlag = new ObTableQueryAndMutateFlag(); - /** + /* * Get pcode. */ @Override @@ -48,7 +50,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_EXECUTE_QUERY; } - /** + /* * Encode. */ @Override @@ -73,11 +75,16 @@ public byte[] encode() { idx += len; System.arraycopy(Serialization.encodeI8(returnAffectedEntity ? (byte) 1 : (byte) 0), 0, bytes, idx, 1); + idx++; + long flags = queryAndMutateFlag.getValue(); + len = Serialization.getNeedBytes(flags); + System.arraycopy(Serialization.encodeVi64(flags), 0, bytes, idx, len); + idx += len; return bytes; } - /** + /* * Decode. */ @Override @@ -94,41 +101,67 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override public long getPayloadContentSize() { return tableQuery.getPayloadSize() // + mutations.getPayloadSize() // - + 1;// returnAffectedEntity + + Serialization.getNeedBytes(queryAndMutateFlag.getValue()) + 1;// returnAffectedEntity } - /** + /* * Get table query. */ public ObTableQuery getTableQuery() { return tableQuery; } - /** + /* * Set table query. */ public void setTableQuery(ObTableQuery tableQuery) { this.tableQuery = tableQuery; } - /** + /* * Get mutations. */ public ObTableBatchOperation getMutations() { return mutations; } - /** + /* * Set mutations. */ public void setMutations(ObTableBatchOperation mutations) { this.mutations = mutations; } + + /* + * Is returning affected entity. + */ + public boolean isReturnAffectedEntity() { + return returnAffectedEntity; + } + + /* + * Set returning affected entity. + */ + public void setReturnAffectedEntity(boolean returnAffectedEntity) { + this.returnAffectedEntity = returnAffectedEntity; + } + + public boolean isReadonly() { + return false; + } + + public void setIsCheckAndExecute(boolean isCheckAndExecute) { + queryAndMutateFlag.setIsCheckAndExecute(isCheckAndExecute); + } + + public void setIsCheckNoExists(boolean isCheckNoExists) { + queryAndMutateFlag.setIsCheckNotExists(isCheckNoExists); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutateRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutateRequest.java index 7121c3f6..490c0619 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutateRequest.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutateRequest.java @@ -17,13 +17,14 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableAbstractOperationRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; -/** +/* * OB_SERIALIZE_MEMBER(ObTableQueryAndMutateRequest, credential_, @@ -38,7 +39,7 @@ public class ObTableQueryAndMutateRequest extends ObTableAbstractOperationReques private ObTableQueryAndMutate tableQueryAndMutate; - /** + /* * Get pcode. */ @Override @@ -46,7 +47,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_QUERY_AND_MUTATE; } - /** + /* * Encode. */ @Override @@ -68,7 +69,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -78,7 +79,10 @@ public Object decode(ByteBuf buf) { this.credential = Serialization.decodeBytesString(buf); this.tableName = Serialization.decodeVString(buf); this.tableId = Serialization.decodeVi64(buf); - this.partitionId = Serialization.decodeVi64(buf); + if (ObGlobal.obVsnMajor() >= 4) + this.partitionId = Serialization.decodeI64(buf); + else + this.partitionId = Serialization.decodeVi64(buf); this.entityType = ObTableEntityType.valueOf(buf.readByte()); this.tableQueryAndMutate = new ObTableQueryAndMutate(); @@ -87,28 +91,49 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override public long getPayloadContentSize() { - return Serialization.getNeedBytes(credential) + Serialization.getNeedBytes(tableName) - + Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(partitionId) + 1 - + tableQueryAndMutate.getPayloadSize(); + if (ObGlobal.obVsnMajor() >= 4) + return Serialization.getNeedBytes(credential) + Serialization.getNeedBytes(tableName) + + Serialization.getNeedBytes(tableId) + 8 + 1 + + tableQueryAndMutate.getPayloadSize(); + else + return Serialization.getNeedBytes(credential) + Serialization.getNeedBytes(tableName) + + Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(partitionId) + + 1 + tableQueryAndMutate.getPayloadSize(); + } - /** + /* * Get table query and mutate. */ public ObTableQueryAndMutate getTableQueryAndMutate() { return tableQueryAndMutate; } - /** + /* * Set table query and mutate. */ public void setTableQueryAndMutate(ObTableQueryAndMutate tableQueryAndMutate) { this.tableQueryAndMutate = tableQueryAndMutate; } + /* + * Is returning affected entity. + */ + @Override + public boolean isReturningAffectedEntity() { + return tableQueryAndMutate.isReturnAffectedEntity(); + } + + /* + * Set returning affected entity. + */ + @Override + public void setReturningAffectedEntity(boolean returningAffectedEntity) { + tableQueryAndMutate.setReturnAffectedEntity(returningAffectedEntity); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutateResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutateResult.java index 6f6aa045..e1d99c17 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutateResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/mutate/ObTableQueryAndMutateResult.java @@ -28,7 +28,7 @@ public class ObTableQueryAndMutateResult extends AbstractPayload { private long affectedRows = 0; private ObTableQueryResult affectedEntity = new ObTableQueryResult(); - /** + /* * Get pcode. */ @Override @@ -36,7 +36,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_QUERY_AND_MUTATE; } - /** + /* * Encode. */ @Override @@ -61,7 +61,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -75,7 +75,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -83,28 +83,28 @@ public long getPayloadContentSize() { return Serialization.getNeedBytes(affectedRows) + affectedEntity.getPayloadSize(); } - /** + /* * Get affected rows. */ public long getAffectedRows() { return affectedRows; } - /** + /* * Set affected rows. */ public void setAffectedRows(long affectedRows) { this.affectedRows = affectedRows; } - /** + /* * Get affected entity. */ public ObTableQueryResult getAffectedEntity() { return affectedEntity; } - /** + /* * Set affected entity. */ public void setAffectedEntity(ObTableQueryResult affectedEntity) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java index 55a86502..6a82d50a 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/AbstractQueryStreamResult.java @@ -17,45 +17,53 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query; -import com.alipay.oceanbase.rpc.exception.FeatureNotSupportedException; -import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.bolt.transport.ObTableConnection; +import com.alipay.oceanbase.rpc.exception.*; import com.alipay.oceanbase.rpc.location.model.ObReadConsistency; +import com.alipay.oceanbase.rpc.location.model.ObServerRoute; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableStreamRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.QueryStreamResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncResult; import com.alipay.oceanbase.rpc.table.ObTable; +import com.alipay.oceanbase.rpc.table.ObTableParam; import io.netty.buffer.ByteBuf; +import org.slf4j.Logger; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; public abstract class AbstractQueryStreamResult extends AbstractPayload implements QueryStreamResult { - private ReentrantLock lock = new ReentrantLock(); - private volatile boolean initialized = false; - private volatile boolean closed = false; - protected volatile List row = null; - protected volatile int rowIndex = -1; - protected ObTableQuery tableQuery; - protected long operationTimeout = -1; - protected String tableName; - protected ObTableEntityType entityType; - private Map> expectant; - private List cacheProperties = new LinkedList(); - private LinkedList> cacheRows = new LinkedList>(); - private LinkedList, ObTableQueryResult>> partitionLastResult = new LinkedList, ObTableQueryResult>>(); - private ObReadConsistency readConsistency = ObReadConsistency.STRONG; - - /** + protected ReentrantLock lock = new ReentrantLock(); + protected volatile boolean initialized = false; + protected volatile boolean closed = false; + protected volatile List row = null; + protected volatile int rowIndex = -1; + protected ObTableQuery tableQuery; + protected long operationTimeout = -1; + protected String tableName; + // use to store the TableEntry Key: + // primary index or local index: key is primary table name + // global index: key is index table name (be like: __idx__) + protected String indexTableName; + protected ObTableEntityType entityType; + protected Map> expectant; // Map> + protected List cacheProperties = new LinkedList(); + protected LinkedList> cacheRows = new LinkedList>(); + private LinkedList, ObTableQueryResult>> partitionLastResult = new LinkedList, ObTableQueryResult>>(); + private ObReadConsistency readConsistency = ObReadConsistency.STRONG; + + /* * Get pcode. */ @Override @@ -63,7 +71,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_EXECUTE_QUERY; } - /** + /* * Encode. */ @Override @@ -71,7 +79,7 @@ public byte[] encode() { throw new FeatureNotSupportedException("stream result can not decode from bytes"); } - /** + /* * Decode. */ @Override @@ -79,7 +87,7 @@ public Object decode(ByteBuf buf) { throw new FeatureNotSupportedException("stream result can not decode from bytes"); } - /** + /* * Get payload content size. */ @Override @@ -87,7 +95,159 @@ public long getPayloadContentSize() { throw new FeatureNotSupportedException("stream result has no pay load size"); } - /** + /* + * Common logic for execute, send the query request to server + */ + protected ObPayload commonExecute(ObTableClient client, Logger logger, + ObPair partIdWithIndex, + ObPayload request, + AtomicReference connectionRef) + throws Exception { + Object result; + ObTable subObTable = partIdWithIndex.getRight().getObTable(); + boolean needRefreshTableEntry = false; + int tryTimes = 0; + long startExecute = System.currentTimeMillis(); + Set failedServerList = null; + ObServerRoute route = null; + while (true) { + client.checkStatus(); + long currentExecute = System.currentTimeMillis(); + long costMillis = currentExecute - startExecute; + if (costMillis > client.getRuntimeMaxWait()) { + long uniqueId = request.getUniqueId(); + long sequence = request.getSequence(); + String trace = String.format("Y%X-%016X", uniqueId, sequence); + throw new ObTableTimeoutExcetion("[" + trace + "]" + " has tried " + tryTimes + + " times and it has waited " + costMillis + + "/ms which exceeds response timeout " + + client.getRuntimeMaxWait() + "/ms"); + } + tryTimes++; + try { + // 重试时重新 getTable + if (tryTimes > 1) { + if (client.isOdpMode()) { + subObTable = client.getOdpTable(); + } else { + if (route == null) { + route = client.getReadRoute(); + } + if (failedServerList != null) { + route.setBlackList(failedServerList); + } + subObTable = client + .getTableWithPartId(indexTableName, partIdWithIndex.getLeft(), + needRefreshTableEntry, client.isTableEntryRefreshIntervalWait(), false, + route).getRight().getObTable(); + } + } + if (client.isOdpMode()) { + result = subObTable.executeWithConnection(request, connectionRef); + } else { + result = subObTable.execute(request); + } + client.resetExecuteContinuousFailureCount(indexTableName); + break; + } catch (Exception e) { + if (client.isOdpMode()) { + if ((tryTimes - 1) < client.getRuntimeRetryTimes()) { + if (e instanceof ObTableException) { + logger + .warn( + "tablename:{} stream query execute while meet Exception needing retry, errorCode: {}, errorMsg: {}, try times {}", + indexTableName, ((ObTableException) e).getErrorCode(), + e.getMessage(), tryTimes); + } else if (e instanceof IllegalArgumentException) { + logger + .warn( + "tablename:{} stream query execute while meet Exception needing retry, try times {}, errorMsg: {}", + indexTableName, tryTimes, e.getMessage()); + } else { + logger + .warn( + "tablename:{} stream query execute while meet Exception needing retry, try times {}", + indexTableName, tryTimes, e); + } + } else { + throw e; + } + } else { + if (e instanceof ObTableReplicaNotReadableException) { + if ((tryTimes - 1) < client.getRuntimeRetryTimes()) { + logger.warn( + "tablename:{} partition id:{} retry when replica not readable: {}", + indexTableName, partIdWithIndex.getLeft(), e.getMessage(), e); + if (failedServerList == null) { + failedServerList = new HashSet(); + } + failedServerList.add(subObTable.getIp()); + } else { + logger + .warn( + "tablename:{} partition id:{} exhaust retry when replica not readable: {}", + indexTableName, partIdWithIndex.getLeft(), e.getMessage(), e); + throw e; + } + } else if (e instanceof ObTableGlobalIndexRouteException) { + if ((tryTimes - 1) < client.getRuntimeRetryTimes()) { + logger + .warn( + "meet global index route expcetion: indexTableName:{} partition id:{}, errorCode: {}, retry times {}", + indexTableName, partIdWithIndex.getLeft(), + ((ObTableException) e).getErrorCode(), tryTimes, e); + indexTableName = client.getIndexTableName(tableName, + tableQuery.getIndexName(), tableQuery.getScanRangeColumns(), true); + } else { + logger + .warn( + "meet global index route expcetion: indexTableName:{} partition id:{}, errorCode: {}, reach max retry times {}", + indexTableName, partIdWithIndex.getLeft(), + ((ObTableException) e).getErrorCode(), tryTimes, e); + throw e; + } + } else if (e instanceof ObTableException) { + if ((((ObTableException) e).getErrorCode() == ResultCodes.OB_TABLE_NOT_EXIST.errorCode || ((ObTableException) e) + .getErrorCode() == ResultCodes.OB_NOT_SUPPORTED.errorCode) + && ((ObTableQueryRequest) request).getTableQuery().isHbaseQuery() + && client.getTableGroupInverted().get(indexTableName) != null) { + // table not exists && hbase mode && table group exists , three condition both + client.eraseTableGroupFromCache(tableName); + } + if (((ObTableException) e).isNeedRefreshTableEntry()) { + needRefreshTableEntry = true; + logger + .warn( + "tablename:{} partition id:{} stream query refresh table while meet Exception needing refresh, errorCode: {}", + indexTableName, partIdWithIndex.getLeft(), + ((ObTableException) e).getErrorCode(), e); + if (client.isRetryOnChangeMasterTimes() + && (tryTimes - 1) < client.getRuntimeRetryTimes()) { + logger + .warn( + "tablename:{} partition id:{} stream query retry while meet Exception needing refresh, errorCode: {} , retry times {}", + indexTableName, partIdWithIndex.getLeft(), + ((ObTableException) e).getErrorCode(), tryTimes, e); + } else { + client.calculateContinuousFailure(indexTableName, e.getMessage()); + throw e; + } + } else { + client.calculateContinuousFailure(indexTableName, e.getMessage()); + throw e; + } + } else { + client.calculateContinuousFailure(indexTableName, e.getMessage()); + throw e; + } + } + } + Thread.sleep(client.getRuntimeRetryInterval()); + } + return (ObPayload) result; + } + + /* * Next. */ public boolean next() throws Exception { @@ -100,7 +260,7 @@ public boolean next() throws Exception { return true; } // secondly, refer to the last stream result - ObPair, ObTableQueryResult> referLastResult; + ObPair, ObTableQueryResult> referLastResult; while ((referLastResult = partitionLastResult.poll()) != null) { ObTableQueryResult lastResult = referLastResult.getRight(); @@ -117,11 +277,12 @@ public boolean next() throws Exception { // lastly, refer to the new partition boolean hasNext = false; - List>> referPartition = new ArrayList>>(); - for (Map.Entry> entry : expectant.entrySet()) { + List>> referPartition = new ArrayList>>(); + for (Map.Entry> entry : expectant.entrySet()) { // mark the refer partition referPartition.add(entry); - ObTableQueryResult tableQueryResult = referToNewPartition(entry.getValue()); + ObTableQueryResult tableQueryResult = (ObTableQueryResult) referToNewPartition(entry + .getValue()); if (tableQueryResult.getRowCount() == 0) { continue; } @@ -131,7 +292,7 @@ public boolean next() throws Exception { } // remove refer partition - for (Map.Entry> entry : referPartition) { + for (Map.Entry> entry : referPartition) { expectant.remove(entry.getKey()); } @@ -141,20 +302,19 @@ public boolean next() throws Exception { } } - private void nextRow() { + protected void nextRow() { rowIndex = rowIndex + 1; row = cacheRows.poll(); } - private void checkStatus() throws IllegalStateException { + protected void checkStatus() throws IllegalStateException { if (!initialized) { throw new IllegalStateException("table " + tableName + "query stream result is not initialized"); } if (closed) { - throw new IllegalStateException("table " + tableName - + " query stream result is closed"); + throw new IllegalStateException("table " + tableName + " query stream result is closed"); } } @@ -171,7 +331,20 @@ protected ObTableQueryResult checkObTableQueryResult(Object result) { return (ObTableQueryResult) result; } - private ObTableQueryResult referToLastStreamResult(ObPair partIdWithObTable, + protected ObTableQueryAsyncResult checkObTableQueryAsyncResult(Object result) { + if (result == null) { + throw new ObTableException("client get unexpected NULL result"); + } + + if (!(result instanceof ObTableQueryAsyncResult)) { + throw new ObTableException("client get unexpected result: " + + result.getClass().getName() + "expect " + + ObTableQueryAsyncResult.class.getName()); + } + return (ObTableQueryAsyncResult) result; + } + + private ObTableQueryResult referToLastStreamResult(ObPair partIdWithObTable, ObTableQueryResult lastResult) throws Exception { ObTableStreamRequest streamRequest = new ObTableStreamRequest(); @@ -180,12 +353,13 @@ private ObTableQueryResult referToLastStreamResult(ObPair partIdW if (operationTimeout > 0) { streamRequest.setTimeout(operationTimeout); } else { - streamRequest.setTimeout(partIdWithObTable.getRight().getObTableOperationTimeout()); + streamRequest.setTimeout(partIdWithObTable.getRight().getObTable() + .getObTableOperationTimeout()); } return execute(partIdWithObTable, streamRequest); } - private void closeLastStreamResult(ObPair partIdWithObTable, + private void closeLastStreamResult(ObPair partIdWithObTable, ObTableQueryResult lastResult) throws Exception { ObTableStreamRequest streamRequest = new ObTableStreamRequest(); streamRequest.setSessionId(lastResult.getSessionId()); @@ -193,44 +367,51 @@ private void closeLastStreamResult(ObPair partIdWithObTable, if (operationTimeout > 0) { streamRequest.setTimeout(operationTimeout); } else { - streamRequest.setTimeout(partIdWithObTable.getRight().getObTableOperationTimeout()); + streamRequest.setTimeout(partIdWithObTable.getRight().getObTable() + .getObTableOperationTimeout()); } - partIdWithObTable.getRight().execute(streamRequest); + partIdWithObTable.getRight().getObTable().execute(streamRequest); } - private ObTableQueryResult referToNewPartition(ObPair partIdWithObTable) - throws Exception { - ObTableQueryRequest request = new ObTableQueryRequest(); - request.setTableName(tableName); - request.setTableQuery(tableQuery); - request.setPartitionId(partIdWithObTable.getLeft()); - request.setEntityType(entityType); - if (operationTimeout > 0) { - request.setTimeout(operationTimeout); - } else { - request.setTimeout(partIdWithObTable.getRight().getObTableOperationTimeout()); - } - request.setConsistencyLevel(getReadConsistency().toObTableConsistencyLevel()); - return execute(partIdWithObTable, request); - } + protected abstract ObPayload referToNewPartition(ObPair partIdWithObTable) + throws Exception; - protected abstract ObTableQueryResult execute(ObPair partIdWithObTable, + protected abstract ObTableQueryResult execute(ObPair partIdWithObTable, ObPayload streamRequest) throws Exception; - private void cacheResultRows(ObTableQueryResult tableQueryResult) { + protected abstract ObTableQueryAsyncResult executeAsync(ObPair partIdWithObTable, + ObPayload streamRequest) + throws Exception; + + protected void cacheResultRows(ObTableQueryResult tableQueryResult) { cacheRows.addAll(tableQueryResult.getPropertiesRows()); cacheProperties = tableQueryResult.getPropertiesNames(); } - protected void cacheStreamNext(ObPair partIdWithObTable, + protected void cacheStreamNext(ObPair partIdWithObTable, ObTableQueryResult tableQueryResult) { cacheResultRows(tableQueryResult); if (tableQueryResult.isStream() && tableQueryResult.isStreamNext()) { - partitionLastResult.addLast(new ObPair, ObTableQueryResult>( + partitionLastResult.addLast(new ObPair, ObTableQueryResult>( partIdWithObTable, tableQueryResult)); } } + private void cacheResultRows(ObTableQueryAsyncResult tableQueryAsyncResult) { + cacheRows.addAll(tableQueryAsyncResult.getAffectedEntity().getPropertiesRows()); + cacheProperties = tableQueryAsyncResult.getAffectedEntity().getPropertiesNames(); + } + + protected void cacheStreamNext(ObPair partIdWithObTable, + ObTableQueryAsyncResult tableQueryAsyncResult) { + cacheResultRows(tableQueryAsyncResult); + if (tableQueryAsyncResult.getAffectedEntity().isStream() + && tableQueryAsyncResult.getAffectedEntity().isStreamNext()) { + partitionLastResult.addLast(new ObPair, ObTableQueryResult>( + partIdWithObTable, tableQueryAsyncResult.getAffectedEntity())); + } + } + /** * Get row. */ @@ -241,7 +422,7 @@ public List getRow() { return row; } - /** + /* * Get row index. */ @Override @@ -249,7 +430,7 @@ public int getRowIndex() { return rowIndex; } - /** + /* * Init. */ @Override @@ -258,11 +439,16 @@ public void init() throws Exception { return; } if (tableQuery.getBatchSize() == -1) { - for (Map.Entry> entry : expectant.entrySet()) { + for (Map.Entry> entry : expectant.entrySet()) { // mark the refer partition referToNewPartition(entry.getValue()); } expectant.clear(); + } else { + // query not support BatchSize + throw new ObTableException( + "simple query not support BatchSize, use executeAsync() instead, BatchSize:" + + tableQuery.getBatchSize()); } initialized = true; } @@ -275,106 +461,120 @@ public void close() throws Exception { return; } closed = true; - ObPair, ObTableQueryResult> referLastResult; + ObPair, ObTableQueryResult> referLastResult; while ((referLastResult = partitionLastResult.poll()) != null) { ObTableQueryResult lastResult = referLastResult.getRight(); closeLastStreamResult(referLastResult.getLeft(), lastResult); } } - /** + /* * Get cache properties. */ public List getCacheProperties() { return cacheProperties; } - /** + /* * Get cache rows. */ public LinkedList> getCacheRows() { return cacheRows; } - public LinkedList, ObTableQueryResult>> getPartitionLastResult() { + public LinkedList, ObTableQueryResult>> getPartitionLastResult() { return partitionLastResult; } - /** + /* * Get table query. */ public ObTableQuery getTableQuery() { return tableQuery; } - /** + /* * Set table query. */ public void setTableQuery(ObTableQuery tableQuery) { this.tableQuery = tableQuery; } - /** + /* * Get operation timeout. */ public long getOperationTimeout() { return operationTimeout; } - /** + /* * Set operation timeout. */ public void setOperationTimeout(long operationTimeout) { this.operationTimeout = operationTimeout; } - /** + /* * Get table name. */ public String getTableName() { return tableName; } - /** + /* * Set table name. */ public void setTableName(String tableName) { this.tableName = tableName; } - /** + /* + * Get index table name. + */ + public String getIndexTableName() { + return indexTableName; + } + + /* + * Set index table name. + */ + public void setIndexTableName(String indexTableName) { + this.indexTableName = indexTableName; + } + + /* * Get entity type. */ public ObTableEntityType getEntityType() { return entityType; } - /** + /* * Set entity type. */ public void setEntityType(ObTableEntityType entityType) { this.entityType = entityType; } - public Map> getExpectant() { + public Map> getExpectant() { return expectant; } - /** + /* * Set expectant. */ - public void setExpectant(Map> expectant) { + public void setExpectant(Map> expectant) { this.expectant = expectant; } - /** + /* * Get Read Consistency */ public ObReadConsistency getReadConsistency() { return readConsistency; } - /** + /* * Set Read Consistency * * @param readConsistency diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObBorderFlag.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObBorderFlag.java index 37235ea6..130c2d46 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObBorderFlag.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObBorderFlag.java @@ -26,7 +26,7 @@ public class ObBorderFlag { private byte value = 0; - /** + /* * Value of. */ public static ObBorderFlag valueOf(byte value) { @@ -35,98 +35,98 @@ public static ObBorderFlag valueOf(byte value) { return obBorderFlag; } - /** + /* * Get value. */ public byte getValue() { return value; } - /** + /* * Set inclusive start. */ public void setInclusiveStart() { this.value |= INCLUSIVE_START; } - /** + /* * Unset inclusive start. */ public void unsetInclusiveStart() { this.value &= (~INCLUSIVE_START); } - /** + /* * Is inclusive start. */ public boolean isInclusiveStart() { return (this.value & INCLUSIVE_START) == INCLUSIVE_START; } - /** + /* * Set inclusive end. */ public void setInclusiveEnd() { this.value |= INCLUSIVE_END; } - /** + /* * Unset inclusive end. */ public void unsetInclusiveEnd() { this.value &= (~INCLUSIVE_END); } - /** + /* * Is inclusive end. */ public boolean isInclusiveEnd() { return (this.value & INCLUSIVE_END) == INCLUSIVE_END; } - /** + /* * Set max value. */ public void setMaxValue() { this.value |= MAX_VALUE; } - /** + /* * Unset max value. */ public void unsetMaxValue() { this.value &= (~MAX_VALUE); } - /** + /* * Is max value. */ public boolean isMaxValue() { return (this.value & MAX_VALUE) == MAX_VALUE; } - /** + /* * Set min value. */ public void setMinValue() { this.value |= MIN_VALUE; } - /** + /* * Unset min value. */ public void unsetMinValue() { this.value &= (~MIN_VALUE); } - /** + /* * Is min value. */ public boolean isMinValue() { return (this.value & MIN_VALUE) == MIN_VALUE; } - /** + /* * Equals. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObHTableFilter.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObHTableFilter.java index 3e16c2ec..02703f1a 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObHTableFilter.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObHTableFilter.java @@ -52,7 +52,7 @@ public class ObHTableFilter extends AbstractPayload { private int offsetPerRowPerCf = 0; // -1 means unlimited private String filterString; - /** + /* * Encode. */ @Override @@ -101,7 +101,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -125,7 +125,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -147,126 +147,126 @@ public long getPayloadContentSize() { return contentSize; } - /** + /* * Is valid. */ public boolean isValid() { return isValid; } - /** + /* * Set valid. */ public void setValid(boolean valid) { isValid = valid; } - /** + /* * Get select column qualifier. */ public List getSelectColumnQualifier() { return selectColumnQualifier; } - /** + /* * Add select column qualifier. */ public void addSelectColumnQualifier(String selectColumnQualifier) { this.selectColumnQualifier.add(new ObBytesString(selectColumnQualifier)); } - /** + /* * Add select column qualifier. */ public void addSelectColumnQualifier(byte[] selectColumnQualifier) { this.selectColumnQualifier.add(new ObBytesString(selectColumnQualifier)); } - /** + /* * Set select column qualifier. */ public void setSelectColumnQualifier(List selectColumnQualifier) { this.selectColumnQualifier = selectColumnQualifier; } - /** + /* * Get min stamp. */ public long getMinStamp() { return minStamp; } - /** + /* * Set min stamp. */ public void setMinStamp(long minStamp) { this.minStamp = minStamp; } - /** + /* * Get max stamp. */ public long getMaxStamp() { return maxStamp; } - /** + /* * Set max stamp. */ public void setMaxStamp(long maxStamp) { this.maxStamp = maxStamp; } - /** + /* * Get max versions. */ public int getMaxVersions() { return maxVersions; } - /** + /* * Set max versions. */ public void setMaxVersions(int maxVersions) { this.maxVersions = maxVersions; } - /** + /* * Get limit per row per cf. */ public int getLimitPerRowPerCf() { return limitPerRowPerCf; } - /** + /* * Set limit per row per cf. */ public void setLimitPerRowPerCf(int limitPerRowPerCf) { this.limitPerRowPerCf = limitPerRowPerCf; } - /** + /* * Get offset per row per cf. */ public int getOffsetPerRowPerCf() { return offsetPerRowPerCf; } - /** + /* * Set offset per row per cf. */ public void setOffsetPerRowPerCf(int offsetPerRowPerCf) { this.offsetPerRowPerCf = offsetPerRowPerCf; } - /** + /* * Get filter string. */ public String getFilterString() { return filterString; } - /** + /* * Set filter string. */ public void setFilterString(String filterString) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObNewRange.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObNewRange.java index a057ca7c..9b54a51b 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObNewRange.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObNewRange.java @@ -17,14 +17,16 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.protocol.payload.Constants; import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; +import com.alipay.oceanbase.rpc.util.ObByteBuf; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; -/** +/* * int ObNewRange::serialize(char *buf, const int64_t buf_len, int64_t &pos) const * { * int ret = OB_SUCCESS; @@ -50,8 +52,9 @@ public class ObNewRange implements ObSimplePayload { private ObBorderFlag borderFlag = new ObBorderFlag(); private ObRowKey startKey; private ObRowKey endKey; + private long flag = 0L; - /** + /* * Ob new range. */ public ObNewRange() { @@ -59,7 +62,7 @@ public ObNewRange() { borderFlag.setInclusiveEnd(); } - /** + /* * Encode. */ @Override @@ -95,10 +98,38 @@ public byte[] encode() { idx += objBytes.length; } + if (ObGlobal.obVsnMajor() >= 4) { + len = Serialization.getNeedBytes(flag); + System.arraycopy(Serialization.encodeVi64(flag), 0, bytes, idx, len); + idx += len; + } + return bytes; } - /** + /* + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVi64(buf, tableId); + Serialization.encodeI8(buf, borderFlag.getValue()); + long startKeyObjCount = startKey.getObjCount(); + Serialization.encodeVi64(buf, startKeyObjCount); + for (int i = 0; i < startKeyObjCount; ++i) { + startKey.getObj(i).encode(buf); + } + long endKeyObjCount = endKey.getObjCount(); + Serialization.encodeVi64(buf, endKeyObjCount); + for (int i = 0; i < endKeyObjCount; ++i) { + endKey.getObj(i).encode(buf); + } + if (ObGlobal.obVsnMajor() >= 4) { + Serialization.encodeVi64(buf, flag); + } + } + + /* * Decode. */ @Override @@ -122,10 +153,14 @@ public Object decode(ByteBuf buf) { this.endKey.addObj(obObj); } + if (ObGlobal.obVsnMajor() >= 4) { + this.flag = Serialization.decodeVi64(buf); + } + return this; } - /** + /* * Get encoded size. */ @Override @@ -147,63 +182,87 @@ public int getEncodedSize() { encodedSize += obObj.getEncodedSize(); } + if (ObGlobal.obVsnMajor() >= 4) { + encodedSize += Serialization.getNeedBytes(flag); + } + return encodedSize; } - /** + /* * Get table id. */ public long getTableId() { return tableId; } - /** + /* * Set table id. */ public void setTableId(long tableId) { this.tableId = tableId; } - /** + /* * Get border flag. */ public ObBorderFlag getBorderFlag() { return borderFlag; } - /** + /* * Set border flag. */ public void setBorderFlag(ObBorderFlag borderFlag) { this.borderFlag = borderFlag; } - /** + /* * Get start key. */ public ObRowKey getStartKey() { return startKey; } - /** + /* * Set start key. */ public void setStartKey(ObRowKey startKey) { this.startKey = startKey; } - /** + /* * Get end key. */ public ObRowKey getEndKey() { return endKey; } - /** + /* * Set end key. */ public void setEndKey(ObRowKey endKey) { this.endKey = endKey; } + /* + * get whole range. + */ + public static ObNewRange getWholeRange() { + ObNewRange range = new ObNewRange(); + range.startKey = ObRowKey.getInstance(ObObj.getMin()); + range.endKey = ObRowKey.getInstance(ObObj.getMax()); + range.borderFlag.unsetInclusiveStart(); + range.borderFlag.unsetInclusiveEnd(); + return range; + } + + public long getFlag() { + return flag; + } + + public void setFlag(long flag) { + this.flag = flag; + } + } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObScanOrder.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObScanOrder.java index cb5e2108..514f1a3c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObScanOrder.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObScanOrder.java @@ -37,21 +37,21 @@ public enum ObScanOrder { } } - /** + /* * Value of. */ public static ObScanOrder valueOf(int value) { return map.get(value); } - /** + /* * Get value. */ public int getValue() { return value; } - /** + /* * Get byte value. */ public byte getByteValue() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java index e0abd062..1dfd2a5c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQuery.java @@ -18,6 +18,8 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query; import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation.ObTableAggregationSingle; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation.ObTableAggregationType; import com.alipay.oceanbase.rpc.util.Serialization; import io.netty.buffer.ByteBuf; @@ -39,7 +41,8 @@ index_name_, batch_size_, max_result_size_, - htable_filter_)); + htable_filter_, + key_range_columns)); * */ public class ObTableQuery extends AbstractPayload { @@ -57,8 +60,35 @@ public class ObTableQuery extends AbstractPayload { private static final byte[] HTABLE_FILTER_DUMMY_BYTES = new byte[] { 0x01, 0x00 }; private boolean isHbaseQuery = false; + private List scanRangeColumns = new LinkedList(); - /** + private List aggregations = new LinkedList<>(); + + /* + * Check filter. + */ + public boolean isFilterNull() { + return filterString != null; + } + + /* + * Check aggregation. + */ + public boolean isAggregation() { + if (aggregations.isEmpty()) { + return false; + } + return true; + } + + /* + * Add aggregation. + */ + public void addAggregation(ObTableAggregationType aggType, String aggColumn) { + this.aggregations.add(new ObTableAggregationSingle(aggType, aggColumn)); + } + + /* * Encode. */ @Override @@ -122,11 +152,31 @@ public byte[] encode() { len = HTABLE_FILTER_DUMMY_BYTES.length; System.arraycopy(HTABLE_FILTER_DUMMY_BYTES, 0, bytes, idx, len); } + idx += len; + + len = Serialization.getNeedBytes(scanRangeColumns.size()); + System.arraycopy(Serialization.encodeVi64(scanRangeColumns.size()), 0, bytes, idx, len); + idx += len; + for (String keyRangeColumn : scanRangeColumns) { + len = Serialization.getNeedBytes(keyRangeColumn); + System.arraycopy(Serialization.encodeVString(keyRangeColumn), 0, bytes, idx, len); + idx += len; + } + + //Aggregation + len = Serialization.getNeedBytes(aggregations.size()); + System.arraycopy(Serialization.encodeVi64(aggregations.size()), 0, bytes, idx, len); + idx += len; + for (ObTableAggregationSingle obTableAggregationSingle : aggregations) { + len = (int) obTableAggregationSingle.getPayloadSize(); + System.arraycopy(obTableAggregationSingle.encode(), 0, bytes, idx, len); + idx += len; + } return bytes; } - /** + /* * Decode. */ @Override @@ -169,11 +219,21 @@ public Object decode(ByteBuf buf) { buf.readByte(); buf.readByte(); } + size = Serialization.decodeVi64(buf); + for (int i = 0; i < size; i++) { + this.scanRangeColumns.add(Serialization.decodeVString(buf)); + } + size = Serialization.decodeVi64(buf); + for (int i = 0; i < size; i++) { + byte agg_type = Serialization.decodeI8(buf); + String agg_column = Serialization.decodeVString(buf); + this.aggregations.add(new ObTableAggregationSingle(ObTableAggregationType.fromByte(agg_type), agg_column)); + } return this; } - /** + /* * Get payload content size. */ @Override @@ -201,129 +261,138 @@ public long getPayloadContentSize() { } else { contentSize += HTABLE_FILTER_DUMMY_BYTES.length; } + contentSize += Serialization.getNeedBytes(scanRangeColumns.size()); + for (String scanRangeColumn : scanRangeColumns) { + contentSize += Serialization.getNeedBytes(scanRangeColumn); + } + + contentSize += Serialization.getNeedBytes(aggregations.size()); + for (ObTableAggregationSingle obTableAggregationSingle : aggregations) { + contentSize += obTableAggregationSingle.getPayloadSize(); + } return contentSize; } - /** + /* * Get key ranges. */ public List getKeyRanges() { return keyRanges; } - /** + /* * Set key ranges. */ public void setKeyRanges(List keyRanges) { this.keyRanges = keyRanges; } - /** + /* * Add key range. */ public void addKeyRange(ObNewRange keyRange) { this.keyRanges.add(keyRange); } - /** + /* * Get select columns. */ public List getSelectColumns() { return selectColumns; } - /** + /* * Set select columns. */ public void setSelectColumns(List selectColumns) { this.selectColumns = selectColumns; } - /** + /* * Add select column. */ public void addSelectColumn(String selectColumn) { this.selectColumns.add(selectColumn); } - /** + /* * Get filter string. */ public String getFilterString() { return filterString; } - /** + /* * Set filter string. */ public void setFilterString(String filterString) { this.filterString = filterString; } - /** + /* * Get limit. */ public int getLimit() { return limit; } - /** + /* * Set limit. */ public void setLimit(int limit) { this.limit = limit; } - /** + /* * Get offset. */ public int getOffset() { return offset; } - /** + /* * Set offset. */ public void setOffset(int offset) { this.offset = offset; } - /** + /* * Get scan order. */ public ObScanOrder getScanOrder() { return scanOrder; } - /** + /* * Set scan order. */ public void setScanOrder(ObScanOrder scanOrder) { this.scanOrder = scanOrder; } - /** + /* * Get index name. */ public String getIndexName() { return indexName; } - /** + /* * Set index name. */ public void setIndexName(String indexName) { this.indexName = indexName; } - /** + /* * Get batch size. */ public int getBatchSize() { return batchSize; } - /** + /* * Set batch size. */ public void setBatchSize(int batchSize) { @@ -332,28 +401,30 @@ public void setBatchSize(int batchSize) { } } - /** + /* * Get max result size. */ public long getMaxResultSize() { return maxResultSize; } - /** + /* * Set max result size. */ public void setMaxResultSize(long maxResultSize) { - this.maxResultSize = maxResultSize; + if (maxResultSize > 0) { + this.maxResultSize = maxResultSize; + } } - /** + /* * Geth table filter. */ public ObHTableFilter gethTableFilter() { return hTableFilter; } - /** + /* * Seth table filter. */ public void sethTableFilter(ObHTableFilter hTableFilter) { @@ -361,17 +432,38 @@ public void sethTableFilter(ObHTableFilter hTableFilter) { this.hTableFilter = hTableFilter; } - /** + /* * Is hbase query. */ public boolean isHbaseQuery() { return isHbaseQuery; } - /** + /* * Set hbase query. */ public void setHbaseQuery(boolean hbaseQuery) { isHbaseQuery = hbaseQuery; } + + /* + * Get select columns. + */ + public List getScanRangeColumns() { + return scanRangeColumns; + } + + /* + * Set select columns. + */ + public void setScanRangeColumns(String... scanRangeColumns) { + this.scanRangeColumns.clear(); + for (String scanRangeCol : scanRangeColumns) { + this.scanRangeColumns.add(scanRangeCol); + } + } + + public void setScanRangeColumns(List scanRangeColumns) { + this.scanRangeColumns = scanRangeColumns; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryRequest.java index c90a9fb8..473ac541 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryRequest.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryRequest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query; +import com.alipay.oceanbase.rpc.ObGlobal; import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableAbstractOperationRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableConsistencyLevel; @@ -41,7 +42,7 @@ public class ObTableQueryRequest extends ObTableAbstractOperationRequest { private ObTableQuery tableQuery; - /** + /* * Get pcode. */ @Override @@ -49,7 +50,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_EXECUTE_QUERY; } - /** + /* * Encode. */ @Override @@ -75,7 +76,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -85,7 +86,10 @@ public Object decode(ByteBuf buf) { this.credential = Serialization.decodeBytesString(buf); this.tableName = Serialization.decodeVString(buf); this.tableId = Serialization.decodeVi64(buf); - this.partitionId = Serialization.decodeVi64(buf); + if (ObGlobal.obVsnMajor() >= 4) + this.partitionId = Serialization.decodeI64(buf); + else + this.partitionId = Serialization.decodeVi64(buf); this.entityType = ObTableEntityType.valueOf(buf.readByte()); this.consistencyLevel = ObTableConsistencyLevel.valueOf(buf.readByte()); @@ -95,24 +99,28 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override public long getPayloadContentSize() { - return Serialization.getNeedBytes(credential) + Serialization.getNeedBytes(tableName) - + Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(partitionId) + 2 - + tableQuery.getPayloadSize(); + if (ObGlobal.obVsnMajor() >= 4) + return Serialization.getNeedBytes(credential) + Serialization.getNeedBytes(tableName) + + Serialization.getNeedBytes(tableId) + 8 + 2 + tableQuery.getPayloadSize(); + else + return Serialization.getNeedBytes(credential) + Serialization.getNeedBytes(tableName) + + Serialization.getNeedBytes(tableId) + Serialization.getNeedBytes(partitionId) + + 2 + tableQuery.getPayloadSize(); } - /** + /* * Get table query. */ public ObTableQuery getTableQuery() { return tableQuery; } - /** + /* * Set table query. */ public void setTableQuery(ObTableQuery tableQuery) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryResult.java index beaa9b56..689315b4 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryResult.java @@ -38,21 +38,21 @@ public class ObTableQueryResult extends AbstractPayload { // TODO 需要做成流式的，目前 OB 还不支持流式协议，单个 packet 大小过大会失败 private List> propertiesRows = new LinkedList>(); - /** + /* * Ob table query result. */ public ObTableQueryResult() { this.header = new ObRpcPacketHeader(); } - /** + /* * Ob table query result. */ public ObTableQueryResult(ObRpcPacketHeader header) { this.header = header; } - /** + /* * Get pcode. */ @Override @@ -60,7 +60,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_EXECUTE_QUERY; } - /** + /* * Encode. */ @Override @@ -103,7 +103,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ @Override @@ -136,7 +136,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -159,91 +159,91 @@ public long getPayloadContentSize() { return size; } - /** + /* * Get properties names. */ public List getPropertiesNames() { return propertiesNames; } - /** + /* * Set properties names. */ public void setPropertiesNames(List propertiesNames) { this.propertiesNames = propertiesNames; } - /** + /* * Add properties name. */ public void addPropertiesName(String propertiesName) { this.propertiesNames.add(propertiesName); } - /** + /* * Get row count. */ public long getRowCount() { return rowCount; } - /** + /* * Set row count. */ public void setRowCount(long rowCount) { this.rowCount = rowCount; } - /** + /* * Get properties rows. */ public List> getPropertiesRows() { return propertiesRows; } - /** + /* * Set properties rows. */ public void setPropertiesRows(List> propertiesRows) { this.propertiesRows = propertiesRows; } - /** + /* * Add properties row. */ public void addPropertiesRow(List propertiesRow) { this.propertiesRows.add(propertiesRow); } - /** + /* * Add all properties rows. */ public void addAllPropertiesRows(List> propertiesRows) { this.propertiesRows.addAll(propertiesRows); } - /** + /* * Is stream. */ public boolean isStream() { return header.isStream(); } - /** + /* * Is stream last. */ public boolean isStreamLast() { return header.isStreamLast(); } - /** + /* * Is stream next. */ public boolean isStreamNext() { return header.isStreamNext(); } - /** + /* * Get session id. */ public long getSessionId() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObQueryOperationType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObQueryOperationType.java new file mode 100644 index 00000000..0f384754 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObQueryOperationType.java @@ -0,0 +1,50 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery; + +import java.util.HashMap; +import java.util.Map; + +public enum ObQueryOperationType { + QUERY_START(0), QUERY_NEXT(1), QUERY_END(2); + + private int value; + private static Map map = new HashMap(); + + ObQueryOperationType(int value) { + this.value = value; + } + + static { + for (ObQueryOperationType type : ObQueryOperationType.values()) { + map.put(type.value, type); + } + } + + public static ObQueryOperationType valueOf(int value) { + return map.get(value); + } + + public int getValue() { + return value; + } + + public byte getByteValue() { + return (byte) value; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObTableQueryAsyncRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObTableQueryAsyncRequest.java new file mode 100644 index 00000000..a8db0e13 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObTableQueryAsyncRequest.java @@ -0,0 +1,127 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery; + +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableAbstractOperationRequest; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryRequest; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +/** + OB_SERIALIZE_MEMBER((ObTableQueryAsyncRequest, + credential_, + table_name_, + table_id_, + partition_id_, + entity_type_, + consistency_level_, + query_, + query_session_id_, + query_type_); + */ +public class ObTableQueryAsyncRequest extends ObTableAbstractOperationRequest { + private ObTableQueryRequest obTableQueryRequest; + private long querySessionId; + private ObQueryOperationType queryType = ObQueryOperationType.QUERY_START; + + /** + * Get pcode. + */ + @Override + public int getPcode() { + return Pcodes.OB_TABLE_API_EXECUTE_QUERY_SYNC; + } + + @Override + public byte[] encode() { + obTableQueryRequest.setCredential(credential); + + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + idx = encodeHeader(bytes, idx); + + int len = (int) obTableQueryRequest.getPayloadSize(); + System.arraycopy(obTableQueryRequest.encode(), 0, bytes, idx, len); + idx += len; + + idx = encodeQuerySessionId(bytes, idx); + + encodeQueryType(bytes, idx); + return bytes; + } + + protected int encodeQuerySessionId(byte[] bytes, int idx) { + int len = Serialization.getNeedBytes(querySessionId); + System.arraycopy(Serialization.encodeVi64(querySessionId), 0, bytes, idx, len); + idx += len; + return idx; + } + + protected int encodeQueryType(byte[] bytes, int idx) { + System.arraycopy(Serialization.encodeI8(queryType.getByteValue()), 0, bytes, idx, 1); + idx += 1; + return idx; + } + + public void setQuerySessionId(long querySessionId) { + this.querySessionId = querySessionId; + } + + public ObQueryOperationType getQueryType() { + return queryType; + } + + public void setQueryType(ObQueryOperationType queryType) { + this.queryType = queryType; + } + + /** + * Decode. + */ + @Override + public Object decode(ByteBuf buf) { + super.decode(buf); + + this.obTableQueryRequest = new ObTableQueryRequest(); + this.obTableQueryRequest.decode(buf); + + this.querySessionId = Serialization.decodeVi64(buf); + this.queryType = ObQueryOperationType.valueOf(buf.readByte()); + + return this; + } + + /** + * Get payload content size. + */ + @Override + public long getPayloadContentSize() { + return Serialization.getNeedBytes(querySessionId) + 1 + + obTableQueryRequest.getPayloadSize(); + } + + public ObTableQueryRequest getObTableQueryRequest() { + return obTableQueryRequest; + } + + public void setObTableQueryRequest(ObTableQueryRequest obTableQueryRequest) { + this.obTableQueryRequest = obTableQueryRequest; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObTableQueryAsyncResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObTableQueryAsyncResult.java new file mode 100644 index 00000000..8ff39714 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/syncquery/ObTableQueryAsyncResult.java @@ -0,0 +1,108 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery; + +import com.alipay.oceanbase.rpc.protocol.payload.AbstractPayload; +import com.alipay.oceanbase.rpc.protocol.payload.Pcodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryResult; +import com.alipay.oceanbase.rpc.util.Serialization; +import io.netty.buffer.ByteBuf; + +/** + OB_SERIALIZE_MEMBER((ObTableQueryAsyncResult, ObTableQueryResult), + is_end_, + query_session_id_); + */ +public class ObTableQueryAsyncResult extends AbstractPayload { + private long sessionId; + private boolean isEnd; + private ObTableQueryResult affectedEntity = new ObTableQueryResult(); + + /** + * Get pcode. + */ + @Override + public int getPcode() { + return Pcodes.OB_TABLE_API_EXECUTE_QUERY_SYNC; + } + + @Override + public byte[] encode() { + byte[] bytes = new byte[(int) getPayloadSize()]; + int idx = 0; + + // 0. encode header + int len = (int) Serialization.getObUniVersionHeaderLength(getVersion(), getPayloadSize()); + byte[] header = Serialization.encodeObUniVersionHeader(getVersion(), getPayloadSize()); + System.arraycopy(header, 0, bytes, idx, len); + idx += len; + + // 1. encode affectedEntity + len = (int) affectedEntity.getPayloadSize(); + System.arraycopy(affectedEntity.encode(), 0, bytes, idx, len); + + // 2. encode isEnd + System.arraycopy(Serialization.encodeI8(isEnd ? (byte) 1 : (byte) 0), 0, bytes, idx, 1); + idx++; + + // 3. encode sessionId + len = Serialization.getNeedBytes(sessionId); + System.arraycopy(Serialization.encodeVi64(sessionId), 0, bytes, idx, len); + idx += len; + + return bytes; + } + + public Object decode(ByteBuf buf) { + super.decode(buf); + this.affectedEntity.decode(buf); + this.isEnd = Serialization.decodeI8(buf) != 0; + this.sessionId = Serialization.decodeVi64(buf); + return this; + } + + @Override + public long getPayloadContentSize() { + return Serialization.getNeedBytes(sessionId) + 3 + affectedEntity.getPayloadContentSize(); + } + + public long getSessionId() { + return sessionId; + } + + public boolean isEnd() { + return isEnd; + } + + public void setEnd(boolean end) { + isEnd = end; + } + + public ObTableQueryResult getAffectedEntity() { + return affectedEntity; + } + + public void setSessionId(long sessionId) { + this.sessionId = sessionId; + } + + public void setAffectedEntity(ObTableQueryResult affectedEntity) { + this.affectedEntity = affectedEntity; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/login/ObTableLoginRequest.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/login/ObTableLoginRequest.java index 7bd74f23..bfdb98a4 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/login/ObTableLoginRequest.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/login/ObTableLoginRequest.java @@ -29,7 +29,7 @@ public class ObTableLoginRequest extends AbstractPayload { - /** + /* * Get pcode. */ @Override @@ -54,13 +54,13 @@ public int getPcode() { private String databaseName; private long ttlUs; - /** + /* * Ob table login request. */ public ObTableLoginRequest() { } - /** + /* * Encode. */ @Override @@ -116,7 +116,7 @@ public byte[] encode() { return bytes; } - /** + /* * Decode. */ public Object decode(byte[] bytes) { @@ -129,7 +129,7 @@ public Object decode(byte[] bytes) { } } - /** + /* * Decode. */ @Override @@ -154,7 +154,7 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get payload content size. */ @Override @@ -167,203 +167,203 @@ public long getPayloadContentSize() { + Serialization.getNeedBytes(databaseName) + Serialization.getNeedBytes(ttlUs); } - /** + /* * Get auth method. */ public byte getAuthMethod() { return authMethod; } - /** + /* * Set auth method. */ public void setAuthMethod(byte authMethod) { this.authMethod = authMethod; } - /** + /* * Get client type. */ public byte getClientType() { return clientType; } - /** + /* * Set client type. */ public void setClientType(byte clientType) { this.clientType = clientType; } - /** + /* * Get client version. */ public byte getClientVersion() { return clientVersion; } - /** + /* * Set client version. */ public void setClientVersion(byte clientVersion) { this.clientVersion = clientVersion; } - /** + /* * Get reversed1. */ public byte getReversed1() { return reversed1; } - /** + /* * Set reversed1. */ public void setReversed1(byte reversed1) { this.reversed1 = reversed1; } - /** + /* * Get client capabilities. */ public int getClientCapabilities() { return clientCapabilities; } - /** + /* * Set client capabilities. */ public void setClientCapabilities(int clientCapabilities) { this.clientCapabilities = clientCapabilities; } - /** + /* * Get max packet size. */ public int getMaxPacketSize() { return maxPacketSize; } - /** + /* * Set max packet size. */ public void setMaxPacketSize(int maxPacketSize) { this.maxPacketSize = maxPacketSize; } - /** + /* * Get reversed2. */ public int getReversed2() { return reversed2; } - /** + /* * Set reversed2. */ public void setReversed2(int reversed2) { this.reversed2 = reversed2; } - /** + /* * Get reversed3. */ public long getReversed3() { return reversed3; } - /** + /* * Set reversed3. */ public void setReversed3(long reversed3) { this.reversed3 = reversed3; } - /** + /* * Get tenant name. */ public String getTenantName() { return tenantName; } - /** + /* * Set tenant name. */ public void setTenantName(String tenantName) { this.tenantName = tenantName; } - /** + /* * Get user name. */ public String getUserName() { return userName; } - /** + /* * Set user name. */ public void setUserName(String userName) { this.userName = userName; } - /** + /* * Get pass secret. */ public ObBytesString getPassSecret() { return passSecret; } - /** + /* * Set pass secret. */ public void setPassSecret(ObBytesString passSecret) { this.passSecret = passSecret; } - /** + /* * Set pass secret. */ public void setPassSecret(String passSecret) { this.passSecret = new ObBytesString(passSecret); } - /** + /* * Get pass scramble. */ public ObBytesString getPassScramble() { return passScramble; } - /** + /* * Set pass scramble. */ public void setPassScramble(ObBytesString passScramble) { this.passScramble = passScramble; } - /** + /* * Get database name. */ public String getDatabaseName() { return databaseName; } - /** + /* * Set database name. */ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } - /** + /* * Get ttl us. */ public long getTtlUs() { return ttlUs; } - /** + /* * Set ttl us. */ public void setTtlUs(long ttlUs) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/login/ObTableLoginResult.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/login/ObTableLoginResult.java index de3dd693..ccd07790 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/login/ObTableLoginResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/login/ObTableLoginResult.java @@ -38,13 +38,13 @@ public class ObTableLoginResult extends AbstractPayload { private long userId; private long databaseId; - /** + /* * Ob table login result. */ public ObTableLoginResult() { } - /** + /* * Get pcode. */ @Override @@ -52,7 +52,7 @@ public int getPcode() { return Pcodes.OB_TABLE_API_LOGIN; } - /** + /* * Encode. */ public byte[] encode() { @@ -93,7 +93,7 @@ public byte[] encode() { return bytes; } - /** + /* * Get payload content size. */ @Override @@ -105,7 +105,7 @@ public long getPayloadContentSize() { + Serialization.getNeedBytes(databaseId); } - /** + /* * Decode. */ @Override @@ -126,112 +126,112 @@ public Object decode(ByteBuf buf) { return this; } - /** + /* * Get server capabilities. */ public int getServerCapabilities() { return serverCapabilities; } - /** + /* * Set server capabilities. */ public void setServerCapabilities(int serverCapabilities) { this.serverCapabilities = serverCapabilities; } - /** + /* * Get reversed1. */ public int getReversed1() { return reversed1; } - /** + /* * Set reversed1. */ public void setReversed1(int reversed1) { this.reversed1 = reversed1; } - /** + /* * Get reversed2. */ public long getReversed2() { return reversed2; } - /** + /* * Set reversed2. */ public void setReversed2(long reversed2) { this.reversed2 = reversed2; } - /** + /* * Get server version. */ public String getServerVersion() { return serverVersion; } - /** + /* * Set server version. */ public void setServerVersion(String serverVersion) { this.serverVersion = serverVersion; } - /** + /* * Get credential. */ public ObBytesString getCredential() { return credential; } - /** + /* * Set credential. */ public void setCredential(ObBytesString credential) { this.credential = credential; } - /** + /* * Get tenant id. */ public long getTenantId() { return tenantId; } - /** + /* * Set tenant id. */ public void setTenantId(long tenantId) { this.tenantId = tenantId; } - /** + /* * Get user id. */ public long getUserId() { return userId; } - /** + /* * Set user id. */ public void setUserId(long userId) { this.userId = userId; } - /** + /* * Get database id. */ public long getDatabaseId() { return databaseId; } - /** + /* * Set database id. */ public void setDatabaseId(long databaseId) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/LayoutCharacters.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/LayoutCharacters.java index 16949002..df0d170a 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/LayoutCharacters.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/LayoutCharacters.java @@ -19,32 +19,32 @@ public interface LayoutCharacters { - /** + /* * Tabulator column increment. */ int TabInc = 8; - /** + /* * Tabulator character. */ byte TAB = 0x8; - /** + /* * Line feed character. */ byte LF = 0xA; - /** + /* * Form feed character. */ byte FF = 0xC; - /** + /* * Carriage return character. */ byte CR = 0xD; - /** + /* * End of input character. Used as a sentinel to denote the character one beyond the last defined character in a * source file. */ diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnCharType.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnCharType.java index 46e5d158..3d38a49f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnCharType.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnCharType.java @@ -34,14 +34,14 @@ public class ObGeneratedColumnCharType { } } - /** + /* * Is hex. */ public static boolean isHex(char c) { return c < 256 && hexFlags[c]; } - /** + /* * Is digit. */ public static boolean isDigit(char c) { @@ -62,7 +62,7 @@ public static boolean isDigit(char c) { firstIdentifierFlags['$'] = true; } - /** + /* * Is first identifier char. */ public static boolean isFirstIdentifierChar(char c) { @@ -89,7 +89,7 @@ public static boolean isFirstIdentifierChar(char c) { identifierFlags['#'] = true; } - /** + /* * Is identifier char. */ public static boolean isIdentifierChar(char c) { @@ -113,7 +113,7 @@ public static boolean isIdentifierChar(char c) { whitespaceFlags[160] = true; // 特别处理 } - /** + /* * @return false if {@link LayoutCharacters#EOI} */ public static boolean isWhitespace(char c) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnExpressParser.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnExpressParser.java index c0d35ccc..b8676737 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnExpressParser.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnExpressParser.java @@ -34,14 +34,14 @@ public class ObGeneratedColumnExpressParser { private final ObGeneratedColumnLexer lexer; - /** + /* * Ob generated column express parser. */ public ObGeneratedColumnExpressParser(String text) { this.lexer = new ObGeneratedColumnLexer(text); } - /** + /* * Parse. */ public ObGeneratedColumnSimpleFunc parse() throws GenerateColumnParseException { @@ -166,7 +166,7 @@ protected Object primaryParameters() { } } - /** + /* * Accept. */ public void accept(ObGeneratedColumnSimpleToken token) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnFuncName.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnFuncName.java index 787fcc5e..937f2214 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnFuncName.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnFuncName.java @@ -37,7 +37,7 @@ public enum ObGeneratedColumnFuncName { } } - /** + /* * Func to token. */ public static ObGeneratedColumnFuncName funcToToken(String func) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnLexer.java b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnLexer.java index 05bde58d..7c47f512 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnLexer.java +++ b/src/main/java/com/alipay/oceanbase/rpc/protocol/payload/impl/parser/ObGeneratedColumnLexer.java @@ -35,7 +35,7 @@ public class ObGeneratedColumnLexer { private int bufPos; protected String stringVal; - /** + /* * Ob generated column lexer. */ public ObGeneratedColumnLexer(String text) { @@ -137,7 +137,7 @@ protected void skipWhitespace() { } } - /** + /* * Scan identifier. */ public void scanIdentifier() { @@ -169,7 +169,7 @@ public void scanIdentifier() { token = IDENTIFIER; } - /** + /* * Scan hexa decimal. */ public void scanHexaDecimal() { @@ -192,7 +192,7 @@ public void scanHexaDecimal() { token = LITERAL_HEX; } - /** + /* * Scan number. */ public void scanNumber() { @@ -276,7 +276,7 @@ private void scanOperator() { } } - /** + /* * Is e o f. */ public boolean isEOF() { @@ -302,7 +302,7 @@ public final String subString(int offset, int count) { } } - /** + /* * Integer value. */ public Number integerValue() { @@ -356,21 +356,21 @@ public Number integerValue() { } } - /** + /* * Token. */ public ObGeneratedColumnSimpleToken token() { return token; } - /** + /* * String val. */ public String stringVal() { return stringVal; } - /** + /* * Pos. */ public int pos() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java new file mode 100644 index 00000000..79096c31 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryAsyncStreamResult.java @@ -0,0 +1,277 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.stream; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.bolt.transport.ObTableConnection; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.location.model.partition.ObPair; +import com.alipay.oceanbase.rpc.protocol.payload.Constants; +import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.AbstractQueryStreamResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryRequest; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObQueryOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncResult; +import com.alipay.oceanbase.rpc.table.ObTableParam; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; + +public class ObTableClientQueryAsyncStreamResult extends AbstractQueryStreamResult { + private static final Logger logger = LoggerFactory + .getLogger(ObTableClientQueryStreamResult.class); + protected ObTableClient client; + private boolean isEnd = true; + private long sessionId = Constants.OB_INVALID_ID; + private ObTableQueryAsyncRequest asyncRequest = new ObTableQueryAsyncRequest(); + private ObTableConnection prevConnection = null; + + @Override + public void init() throws Exception { + if (initialized) { + return; + } + // init request + ObTableQueryRequest request = new ObTableQueryRequest(); + request.setTableName(tableName); + request.setTableQuery(tableQuery); + request.setEntityType(entityType); + request.setConsistencyLevel(getReadConsistency().toObTableConsistencyLevel()); + + // construct async query request + asyncRequest.setObTableQueryRequest(request); + asyncRequest.setQueryType(ObQueryOperationType.QUERY_START); + asyncRequest.setQuerySessionId(Constants.OB_INVALID_ID); + + // send to first tablet + if (!expectant.isEmpty()) { + Iterator>> it = expectant.entrySet() + .iterator(); + Map.Entry> firstEntry = it.next(); + referToNewPartition(firstEntry.getValue()); + if (isEnd()) + it.remove(); + } + initialized = true; + } + + protected void cacheResultRows(ObTableQueryAsyncResult tableQueryResult) { + cacheRows.clear(); + cacheRows.addAll(tableQueryResult.getAffectedEntity().getPropertiesRows()); + cacheProperties = tableQueryResult.getAffectedEntity().getPropertiesNames(); + } + + protected ObTableQueryAsyncResult referToNewPartition(ObPair partIdWithObTable) + throws Exception { + ObTableParam obTableParam = partIdWithObTable.getRight(); + ObTableQueryRequest queryRequest = asyncRequest.getObTableQueryRequest(); + + // refresh request info + queryRequest.setPartitionId(obTableParam.getPartitionId()); + queryRequest.setTableId(obTableParam.getTableId()); + if (operationTimeout > 0) { + queryRequest.setTimeout(operationTimeout); + } else { + queryRequest.setTimeout(obTableParam.getObTable().getObTableOperationTimeout()); + } + + // refresh async query request + // since we try to access a new server, we set corresponding status + asyncRequest.setQueryType(ObQueryOperationType.QUERY_START); + asyncRequest.setQuerySessionId(Constants.OB_INVALID_ID); + + // async execute + ObTableQueryAsyncResult ret = executeAsync(partIdWithObTable, asyncRequest); + + return ret; + } + + protected ObTableQueryAsyncResult referToLastStreamResult(ObPair partIdWithObTable) + throws Exception { + ObTableParam obTableParam = partIdWithObTable.getRight(); + ObTableQueryRequest queryRequest = asyncRequest.getObTableQueryRequest(); + + // refresh request info + queryRequest.setPartitionId(obTableParam.getPartitionId()); + queryRequest.setTableId(obTableParam.getTableId()); + + // refresh async query request + asyncRequest.setQueryType(ObQueryOperationType.QUERY_NEXT); + asyncRequest.setQuerySessionId(sessionId); + + // async execute + ObTableQueryAsyncResult ret = executeAsync(partIdWithObTable, asyncRequest); + + return ret; + } + + protected void closeLastStreamResult(ObPair partIdWithObTable) + throws Exception { + ObTableParam obTableParam = partIdWithObTable.getRight(); + ObTableQueryRequest queryRequest = asyncRequest.getObTableQueryRequest(); + + // refresh request info + queryRequest.setPartitionId(obTableParam.getPartitionId()); + queryRequest.setTableId(obTableParam.getTableId()); + + // set end async query + asyncRequest.setQueryType(ObQueryOperationType.QUERY_END); + asyncRequest.setQuerySessionId(sessionId); + + // async execute + ObTableQueryAsyncResult ret = executeAsync(partIdWithObTable, asyncRequest); + + if (!isEnd()) { + throw new ObTableException("failed to close last stream result"); + } + } + + @Override + public boolean next() throws Exception { + checkStatus(); + lock.lock(); + try { + // firstly, refer to the cache + if (!cacheRows.isEmpty()) { + nextRow(); + return true; + } + + // secondly, refer to the last stream result + if (!isEnd() && !expectant.isEmpty()) { + Iterator>> it = expectant.entrySet() + .iterator(); + Map.Entry> lastEntry = it.next(); + // try access new partition, async will not remove useless expectant + referToLastStreamResult(lastEntry.getValue()); + + // remove useless expectant if it is end + if (isEnd()) + it.remove(); + + if (!cacheRows.isEmpty()) { + nextRow(); + return true; + } + } + + // lastly, refer to the new partition + boolean hasNext = false; + Iterator>> it = expectant.entrySet() + .iterator(); + while (it.hasNext()) { + Map.Entry> entry = it.next(); + // try access new partition, async will not remove useless expectant + referToNewPartition(entry.getValue()); + + // remove useless expectant if it is end + if (isEnd()) + it.remove(); + + if (!cacheRows.isEmpty()) { + hasNext = true; + nextRow(); + break; + } + } + + return hasNext; + } finally { + lock.unlock(); + } + } + + @Override + protected ObTableQueryResult execute(ObPair partIdWithObTable, + ObPayload streamRequest) throws Exception { + throw new IllegalArgumentException("not support this execute"); + } + + /* + * Attention: executeAsync will remove useless expectant + */ + @Override + protected ObTableQueryAsyncResult executeAsync(ObPair partIdWithObTable, + ObPayload streamRequest) throws Exception { + // Construct connection reference + AtomicReference connectionRef = new AtomicReference<>(); + if (client.isOdpMode() && !isEnd && prevConnection != null) { + connectionRef.set(prevConnection); + } + + // execute request + ObTableQueryAsyncResult result = (ObTableQueryAsyncResult) commonExecute(this.client, + logger, partIdWithObTable, streamRequest, connectionRef); + + // cache result + cacheResultRows(result); + + // refresh async query status + if (result.isEnd()) { + isEnd = true; + } else { + isEnd = false; + prevConnection = connectionRef.get(); + } + sessionId = result.getSessionId(); + return result; + } + + @Override + public void close() throws Exception { + // set status + if (closed) { + return; + } + closed = true; + + // send end packet to last tablet + if (!isEnd() && !expectant.isEmpty()) { + Iterator>> it = expectant.entrySet() + .iterator(); + // get the last tablet + Map.Entry> lastEntry = it.next(); + // try access new partition, async will not remove useless expectant + closeLastStreamResult(lastEntry.getValue()); + } + } + + public ObTableClient getClient() { + return client; + } + + /** + * Set client. + * @param client client want to set + */ + public void setClient(ObTableClient client) { + this.client = client; + } + + public boolean isEnd() { + return isEnd; + } + + public void setEnd(boolean end) { + isEnd = end; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java index bd1759ad..ef55baeb 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableClientQueryStreamResult.java @@ -18,20 +18,18 @@ package com.alipay.oceanbase.rpc.stream; import com.alipay.oceanbase.rpc.ObTableClient; -import com.alipay.oceanbase.rpc.exception.ObTableException; -import com.alipay.oceanbase.rpc.exception.ObTableReplicaNotReadableException; -import com.alipay.oceanbase.rpc.exception.ObTableTimeoutExcetion; -import com.alipay.oceanbase.rpc.location.model.ObServerRoute; +import com.alipay.oceanbase.rpc.bolt.transport.ObTableConnection; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.AbstractQueryStreamResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryRequest; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryResult; -import com.alipay.oceanbase.rpc.table.ObTable; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncResult; +import com.alipay.oceanbase.rpc.table.ObTableParam; import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; import org.slf4j.Logger; -import java.util.HashSet; -import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; public class ObTableClientQueryStreamResult extends AbstractQueryStreamResult { @@ -39,92 +37,54 @@ public class ObTableClientQueryStreamResult extends AbstractQueryStreamResult { .getLogger(ObTableClientQueryStreamResult.class); protected ObTableClient client; - protected ObTableQueryResult execute(ObPair partIdWithIndex, ObPayload request) + protected ObTableQueryResult referToNewPartition(ObPair partIdWithObTable) throws Exception { - Object result; - ObTable subObTable = partIdWithIndex.getRight(); - boolean needRefreshTableEntry = false; - int tryTimes = 0; - long startExecute = System.currentTimeMillis(); - Set failedServerList = null; - ObServerRoute route = null; - while (true) { - client.checkStatus(); - long currentExecute = System.currentTimeMillis(); - long costMillis = currentExecute - startExecute; - if (costMillis > client.getRuntimeMaxWait()) { - throw new ObTableTimeoutExcetion("it has tried " + tryTimes - + " times and it has waited " + costMillis - + "/ms which exceeds response timeout " - + client.getRuntimeMaxWait() + "/ms"); - } - tryTimes++; - try { - // 重试时重新 getTable - if (tryTimes > 1) { - if (route == null) { - route = client.getReadRoute(); - } - if (failedServerList != null) { - route.setBlackList(failedServerList); - } - subObTable = client.getTable(tableName, partIdWithIndex.getLeft(), - needRefreshTableEntry, client.isTableEntryRefreshIntervalWait(), route) - .getRight(); - } - result = subObTable.execute(request); - client.resetExecuteContinuousFailureCount(tableName); - break; - } catch (ObTableReplicaNotReadableException ex) { - if ((tryTimes - 1) < client.getRuntimeRetryTimes()) { - logger.warn("retry when replica not readable: {}", ex.getMessage()); - if (failedServerList == null) { - failedServerList = new HashSet(); - } - failedServerList.add(subObTable.getIp()); - } else { - logger.warn("exhaust retry when replica not readable: {}", ex.getMessage()); - throw ex; - } - } catch (Exception e) { - if (e instanceof ObTableException - && ((ObTableException) e).isNeedRefreshTableEntry()) { - needRefreshTableEntry = true; - logger - .warn( - "stream query refresh table while meet ObTableMasterChangeException, errorCode: {}", - ((ObTableException) e).getErrorCode()); - if (client.isRetryOnChangeMasterTimes() - && (tryTimes - 1) < client.getRuntimeRetryTimes()) { - logger - .warn( - "stream query retry while meet ObTableMasterChangeException, errorCode: {} , retry times {}", - ((ObTableException) e).getErrorCode(), tryTimes); - } else { - client.calculateContinuousFailure(tableName, e.getMessage()); - throw e; - } - } else { - client.calculateContinuousFailure(tableName, e.getMessage()); - throw e; - } - } - Thread.sleep(client.getRuntimeRetryInterval()); + ObTableQueryRequest request = new ObTableQueryRequest(); + request.setTableName(tableName); + request.setTableQuery(tableQuery); + request.setPartitionId(partIdWithObTable.getRight().getPartitionId()); + request.setTableId(partIdWithObTable.getRight().getTableId()); + request.setEntityType(entityType); + if (operationTimeout > 0) { + request.setTimeout(operationTimeout); + } else { + request.setTimeout(partIdWithObTable.getRight().getObTable() + .getObTableOperationTimeout()); } + request.setConsistencyLevel(getReadConsistency().toObTableConsistencyLevel()); + return execute(partIdWithObTable, request); + } + + @Override + protected ObTableQueryResult execute(ObPair partIdWithIndex, + ObPayload request) throws Exception { + // Construct connection reference (useless in sync query) + AtomicReference connectionRef = new AtomicReference<>(); + + // execute request + ObTableQueryResult result = (ObTableQueryResult) commonExecute(this.client, logger, + partIdWithIndex, request, connectionRef); cacheStreamNext(partIdWithIndex, checkObTableQueryResult(result)); - return (ObTableQueryResult) result; + return result; + } + + @Override + protected ObTableQueryAsyncResult executeAsync(ObPair partIdWithObTable, + ObPayload streamRequest) throws Exception { + throw new IllegalArgumentException("not support this execute"); } /** * Get client. + * @return client */ public ObTableClient getClient() { return client; } - /** + /* * Set client. */ public void setClient(ObTableClient client) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableQueryStreamResult.java b/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableQueryStreamResult.java deleted file mode 100644 index 7cd78344..00000000 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/ObTableQueryStreamResult.java +++ /dev/null @@ -1,36 +0,0 @@ -/*- - * #%L - * OBKV Table Client Framework - * %% - * Copyright (C) 2021 OceanBase - * %% - * OBKV Table Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.rpc.stream; - -import com.alipay.oceanbase.rpc.location.model.partition.ObPair; -import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.AbstractQueryStreamResult; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryResult; -import com.alipay.oceanbase.rpc.table.ObTable; - -public class ObTableQueryStreamResult extends AbstractQueryStreamResult { - - protected ObTableQueryResult execute(ObPair partIdWithIndex, ObPayload request) - throws Exception { - Object result = partIdWithIndex.getRight().execute(request); - - cacheStreamNext(partIdWithIndex, checkObTableQueryResult(result)); - - return (ObTableQueryResult) result; - } -} diff --git a/src/main/java/com/alipay/oceanbase/rpc/stream/QueryResultSet.java b/src/main/java/com/alipay/oceanbase/rpc/stream/QueryResultSet.java index 56e7806f..29df3998 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/stream/QueryResultSet.java +++ b/src/main/java/com/alipay/oceanbase/rpc/stream/QueryResultSet.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.rpc.stream; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.QueryStreamResult; @@ -27,7 +28,7 @@ public class QueryResultSet { private final QueryStreamResult queryStreamResult; - /** + /* * Query result set. */ public QueryResultSet(QueryStreamResult queryStreamResult) { @@ -38,7 +39,7 @@ public QueryStreamResult getQueryStreamResult() { return queryStreamResult; } - /** + /* * Next. */ public boolean next() throws Exception { @@ -56,18 +57,24 @@ public Map getRow() { return rowValue; } - /** + /* + * get Row with Row result + */ + public Row getResultRow() { + return new Row(getRow()); + } + + /* * Cache size. */ public int cacheSize() { return queryStreamResult.getCacheRows().size(); } - /** + /* * Close. */ public void close() throws Exception { queryStreamResult.close(); } - } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractObTable.java b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractObTable.java index a035b4c6..a43bc222 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractObTable.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractObTable.java @@ -41,119 +41,119 @@ public abstract class AbstractObTable extends AbstractTable { protected int nettyBlockingWaitInterval = NETTY_BLOCKING_WAIT_INTERVAL.getDefaultInt(); - /** + /* * Get ob table connect try times. */ public int getObTableConnectTryTimes() { return obTableConnectTryTimes; } - /** + /* * Set ob table connect try times. */ public void setObTableConnectTryTimes(int obTableConnectTryTimes) { this.obTableConnectTryTimes = obTableConnectTryTimes; } - /** + /* * Get ob table login try times. */ public int getObTableLoginTryTimes() { return obTableLoginTryTimes; } - /** + /* * Set ob table login try times. */ public void setObTableLoginTryTimes(int obTableLoginTryTimes) { this.obTableLoginTryTimes = obTableLoginTryTimes; } - /** + /* * Get ob table connection pool size. */ public int getObTableConnectionPoolSize() { return obTableConnectionPoolSize; } - /** + /* * Set ob table connection pool size. */ public void setObTableConnectionPoolSize(int obTableConnectionPoolSize) { this.obTableConnectionPoolSize = obTableConnectionPoolSize; } - /** + /* * Get ob table connect timeout. */ public int getObTableConnectTimeout() { return obTableConnectTimeout; } - /** + /* * Set ob table connect timeout. */ public void setObTableConnectTimeout(int obTableConnectTimeout) { this.obTableConnectTimeout = obTableConnectTimeout; } - /** + /* * Get ob table execute timeout. */ public int getObTableExecuteTimeout() { return obTableExecuteTimeout; } - /** + /* * Set ob table execute timeout. */ public void setObTableExecuteTimeout(int obTableExecuteTimeout) { this.obTableExecuteTimeout = obTableExecuteTimeout; } - /** + /* * Get ob table login timeout. */ public int getObTableLoginTimeout() { return obTableLoginTimeout; } - /** + /* * Set ob table login timeout. */ public void setObTableLoginTimeout(int obTableLoginTimeout) { this.obTableLoginTimeout = obTableLoginTimeout; } - /** + /* * Get ob table operation timeout. */ public long getObTableOperationTimeout() { return obTableOperationTimeout; } - /** + /* * Set ob table operation timeout. */ public void setObTableOperationTimeout(long obTableOperationTimeout) { this.obTableOperationTimeout = obTableOperationTimeout; } - /** + /* * Get netty buffer low watermark. */ public int getNettyBufferLowWatermark() { return nettyBufferLowWatermark; } - /** + /* * Get netty buffer high watermark. */ public int getNettyBufferHighWatermark() { return nettyBufferHighWatermark; } - /** + /* * Get netty blocking wait interval. */ public int getNettyBlockingWaitInterval() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractObTableClient.java index dbaee67e..e701af5f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractObTableClient.java @@ -56,7 +56,7 @@ public abstract class AbstractObTableClient extends AbstractTable { protected boolean tableEntryRefreshIntervalWait = TABLE_ENTRY_REFRESH_INTERVAL_WAIT .getDefaultBoolean(); - protected int tableEntryRefreshTryTimes = TABLE_ENTRY_REFRESH_TYE_TIMES + protected int tableEntryRefreshTryTimes = TABLE_ENTRY_REFRESH_TRY_TIMES .getDefaultInt(); protected long tableEntryRefreshLockTimeout = TABLE_ENTRY_REFRESH_LOCK_TIMEOUT @@ -92,9 +92,11 @@ public abstract class AbstractObTableClient extends AbstractTable { .getDefaultInt(); protected int rpcLoginTimeout = RPC_LOGIN_TIMEOUT .getDefaultInt(); + protected long slowQueryMonitorThreshold = SLOW_QUERY_MONITOR_THRESHOLD + .getDefaultLong(); @Deprecated - /** + /* * Get metadata refresh internal. */ public long getMetadataRefreshInternal() { @@ -102,77 +104,77 @@ public long getMetadataRefreshInternal() { } @Deprecated - /** + /* * Set metadata refresh internal. */ public void setMetadataRefreshInternal(long metadataRefreshInterval) { this.metadataRefreshInterval = metadataRefreshInterval; } - /** + /* * Get metadata refresh interval. */ public long getMetadataRefreshInterval() { return metadataRefreshInterval; } - /** + /* * Set metadata refresh interval. */ public void setMetadataRefreshInterval(long metadataRefreshInterval) { this.metadataRefreshInterval = metadataRefreshInterval; } - /** + /* * Get metadata refresh lock timeout. */ public long getMetadataRefreshLockTimeout() { return metadataRefreshLockTimeout; } - /** + /* * Set metadata refresh lock timeout. */ public void setMetadataRefreshLockTimeout(long metadataRefreshLockTimeout) { this.metadataRefreshLockTimeout = metadataRefreshLockTimeout; } - /** + /* * Get rs list acquire connect timeout. */ public int getRsListAcquireConnectTimeout() { return rsListAcquireConnectTimeout; } - /** + /* * Set rs list acquire connect timeout. */ public void setRsListAcquireConnectTimeout(int rsListAcquireConnectTimeout) { this.rsListAcquireConnectTimeout = rsListAcquireConnectTimeout; } - /** + /* * Get rs list acquire read timeout. */ public int getRsListAcquireReadTimeout() { return rsListAcquireReadTimeout; } - /** + /* * Set rs list acquire read timeout. */ public void setRsListAcquireReadTimeout(int rsListAcquireReadTimeout) { this.rsListAcquireReadTimeout = rsListAcquireReadTimeout; } - /** + /* * Get rs list acquire try times. */ public int getRsListAcquireTryTimes() { return rsListAcquireTryTimes; } - /** + /* * Set rs list acquire try times. */ public void setRsListAcquireTryTimes(int rsListAcquireTryTimes) { @@ -180,7 +182,7 @@ public void setRsListAcquireTryTimes(int rsListAcquireTryTimes) { } @Deprecated - /** + /* * Get rs list acquire retry internal. */ public long getRsListAcquireRetryInternal() { @@ -188,234 +190,234 @@ public long getRsListAcquireRetryInternal() { } @Deprecated - /** + /* * Set rs list acquire retry internal. */ public void setRsListAcquireRetryInternal(long rsListAcquireRetryInterval) { this.rsListAcquireRetryInterval = rsListAcquireRetryInterval; } - /** + /* * Get rs list acquire retry interval. */ public long getRsListAcquireRetryInterval() { return rsListAcquireRetryInterval; } - /** + /* * Set rs list acquire retry interval. */ public void setRsListAcquireRetryInterval(long rsListAcquireRetryInterval) { this.rsListAcquireRetryInterval = rsListAcquireRetryInterval; } - /** + /* * Get table entry acquire connect timeout. */ public long getTableEntryAcquireConnectTimeout() { return tableEntryAcquireConnectTimeout; } - /** + /* * Set table entry acquire connect timeout. */ public void setTableEntryAcquireConnectTimeout(long tableEntryAcquireConnectTimeout) { this.tableEntryAcquireConnectTimeout = tableEntryAcquireConnectTimeout; } - /** + /* * Get table entry acquire socket timeout. */ public long getTableEntryAcquireSocketTimeout() { return tableEntryAcquireSocketTimeout; } - /** + /* * Set table entry acquire socket timeout. */ public void setTableEntryAcquireSocketTimeout(long tableEntryAcquireSocketTimeout) { this.tableEntryAcquireSocketTimeout = tableEntryAcquireSocketTimeout; } - /** + /* * Get table entry refresh interval base. */ public long getTableEntryRefreshIntervalBase() { return tableEntryRefreshIntervalBase; } - /** + /* * Set table entry refresh interval base. */ public void setTableEntryRefreshIntervalBase(long tableEntryRefreshIntervalBase) { this.tableEntryRefreshIntervalBase = tableEntryRefreshIntervalBase; } - /** + /* * Get table entry refresh interval ceiling. */ public long getTableEntryRefreshIntervalCeiling() { return tableEntryRefreshIntervalCeiling; } - /** + /* * Set table entry refresh interval ceiling. */ public void setTableEntryRefreshIntervalCeiling(long tableEntryRefreshIntervalCeiling) { this.tableEntryRefreshIntervalCeiling = tableEntryRefreshIntervalCeiling; } - /** + /* * Get table entry refresh try times. */ public int getTableEntryRefreshTryTimes() { return tableEntryRefreshTryTimes; } - /** + /* * Set table entry refresh try times. */ public void setTableEntryRefreshTryTimes(int tableEntryRefreshTryTimes) { this.tableEntryRefreshTryTimes = tableEntryRefreshTryTimes; } - /** + /* * Get table entry refresh lock timeout. */ public long getTableEntryRefreshLockTimeout() { return tableEntryRefreshLockTimeout; } - /** + /* * Set table entry refresh lock timeout. */ public void setTableEntryRefreshLockTimeout(long tableEntryRefreshLockTimeout) { this.tableEntryRefreshLockTimeout = tableEntryRefreshLockTimeout; } - /** + /* * Get table entry refresh continuous failure ceiling. */ public int getTableEntryRefreshContinuousFailureCeiling() { return tableEntryRefreshContinuousFailureCeiling; } - /** + /* * Set table entry refresh continuous failure ceiling. */ public void setTableEntryRefreshContinuousFailureCeiling(int tableEntryRefreshContinuousFailureCeiling) { this.tableEntryRefreshContinuousFailureCeiling = tableEntryRefreshContinuousFailureCeiling; } - /** + /* * Get server address priority timeout. */ public long getServerAddressPriorityTimeout() { return serverAddressPriorityTimeout; } - /** + /* * Set server address priority timeout. */ public void setServerAddressPriorityTimeout(long serverAddressPriorityTimeout) { this.serverAddressPriorityTimeout = serverAddressPriorityTimeout; } - /** + /* * Get server address caching timeout. */ public long getServerAddressCachingTimeout() { return serverAddressCachingTimeout; } - /** + /* * Set server address caching timeout. */ public void setServerAddressCachingTimeout(long serverAddressCachingTimeout) { this.serverAddressCachingTimeout = serverAddressCachingTimeout; } - /** + /* * Get runtime continuous failure ceiling. */ public int getRuntimeContinuousFailureCeiling() { return runtimeContinuousFailureCeiling; } - /** + /* * Set runtime continuous failure ceiling. */ public void setRuntimeContinuousFailureCeiling(int runtimeContinuousFailureCeiling) { this.runtimeContinuousFailureCeiling = runtimeContinuousFailureCeiling; } - /** + /* * Get runtime max wait. */ public long getRuntimeMaxWait() { return runtimeMaxWait; } - /** + /* * Set runtime max wait. */ public void setRuntimeMaxWait(long runtimeMaxWait) { this.runtimeMaxWait = runtimeMaxWait; + this.properties.put(RUNTIME_MAX_WAIT.getKey(), String.valueOf(runtimeMaxWait)); + } - /** + /* * Get runtime retry times. */ public int getRuntimeRetryTimes() { return runtimeRetryTimes; } - /** + /* * Get runtime retry interval. */ public int getRuntimeRetryInterval() { return runtimeRetryInterval; } - /** + /* * Set runtime retry interval. */ public void setRuntimeRetryInterval(int runtimeRetryInterval) { this.runtimeRetryInterval = runtimeRetryInterval; } - /** + /* * Set runtime retry times. */ public void setRuntimeRetryTimes(int runtimeRetryTimes) { - // ignore the illegal param - if (runtimeRetryTimes >= 1) { - this.runtimeRetryTimes = runtimeRetryTimes; - } + this.properties.put(RUNTIME_RETRY_TIMES.getKey(), String.valueOf(runtimeRetryTimes)); + this.runtimeRetryTimes = runtimeRetryTimes; } - /** + /* * Is table entry refresh interval wait. */ public boolean isTableEntryRefreshIntervalWait() { return tableEntryRefreshIntervalWait; } - /** + /* * Set table entry refresh interval wait. */ public void setTableEntryRefreshIntervalWait(boolean tableEntryRefreshIntervalWait) { this.tableEntryRefreshIntervalWait = tableEntryRefreshIntervalWait; } - /** + /* * Get rpc connect timeout. */ public int getRpcConnectTimeout() { return rpcConnectTimeout; } - /** + /* * Set rpc connect timeout. */ public void setRpcConnectTimeout(int rpcConnectTimeout) { @@ -423,14 +425,14 @@ public void setRpcConnectTimeout(int rpcConnectTimeout) { this.rpcConnectTimeout = rpcConnectTimeout; } - /** + /* * Get rpc execute timeout. */ public int getRpcExecuteTimeout() { return rpcExecuteTimeout; } - /** + /* * Set rpc execute timeout. */ public void setRpcExecuteTimeout(int rpcExecuteTimeout) { @@ -438,14 +440,14 @@ public void setRpcExecuteTimeout(int rpcExecuteTimeout) { this.rpcExecuteTimeout = rpcExecuteTimeout; } - /** + /* * Get rpc login timeout. */ public int getRpcLoginTimeout() { return rpcLoginTimeout; } - /** + /* * Set rpc login timeout. */ public void setRpcLoginTimeout(int rpcLoginTimeout) { @@ -453,31 +455,48 @@ public void setRpcLoginTimeout(int rpcLoginTimeout) { this.rpcLoginTimeout = rpcLoginTimeout; } - /** + /* * Get runtime batch max wait. */ public long getRuntimeBatchMaxWait() { return runtimeBatchMaxWait; } - /** + /* * Set runtime batch max wait. */ public void setRuntimeBatchMaxWait(long runtimeBatchMaxWait) { + this.properties.put(RUNTIME_BATCH_MAX_WAIT.getKey(), String.valueOf(runtimeBatchMaxWait)); this.runtimeBatchMaxWait = runtimeBatchMaxWait; } - /** + /* * Get runtime batch executor. */ public ExecutorService getRuntimeBatchExecutor() { return runtimeBatchExecutor; } - /** + /* * Set runtime batch executor. */ public void setRuntimeBatchExecutor(ExecutorService runtimeBatchExecutor) { this.runtimeBatchExecutor = runtimeBatchExecutor; } + + /* + * Get slow query threshold. + */ + public long getslowQueryMonitorThreshold() { + return slowQueryMonitorThreshold; + } + + /* + * Set slow query threshold. + */ + public void setslowQueryMonitorThreshold(long slowQueryMonitorThreshold) { + this.properties.put(SLOW_QUERY_MONITOR_THRESHOLD.getKey(), + String.valueOf(slowQueryMonitorThreshold)); + this.slowQueryMonitorThreshold = slowQueryMonitorThreshold; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTable.java b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTable.java index 3e84abc5..724e8a29 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTable.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTable.java @@ -29,7 +29,7 @@ public Map get(String tableName, Object rowkey, String[] columns return get(tableName, new Object[] { rowkey }, columns); } - /** + /* * Update. */ public long update(String tableName, Object rowkey, String[] columns, Object[] values) @@ -37,14 +37,14 @@ public long update(String tableName, Object rowkey, String[] columns, Object[] v return update(tableName, new Object[] { rowkey }, columns, values); } - /** + /* * Delete. */ public long delete(String tableName, Object rowkey) throws Exception { return delete(tableName, new Object[] { rowkey }); } - /** + /* * Insert. */ public long insert(String tableName, Object rowkey, String[] columns, Object[] values) @@ -52,7 +52,7 @@ public long insert(String tableName, Object rowkey, String[] columns, Object[] v return insert(tableName, new Object[] { rowkey }, columns, values); } - /** + /* * Replace. */ public long replace(String tableName, Object rowkey, String[] columns, Object[] values) @@ -60,7 +60,7 @@ public long replace(String tableName, Object rowkey, String[] columns, Object[] return replace(tableName, new Object[] { rowkey }, columns, values); } - /** + /* * Insert or update. */ public long insertOrUpdate(String tableName, Object rowkey, String[] columns, Object[] values) diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java index 783b2bf1..85b5c4ad 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableBatchOps.java @@ -25,8 +25,9 @@ public abstract class AbstractTableBatchOps implements TableBatchOps { protected String tableName; protected boolean atomicOperation; + protected boolean returnOneResult; - protected ObTableEntityType entityType = ObTableEntityType.DYNAMIC; + protected ObTableEntityType entityType = ObTableEntityType.KV; /** * Get. @@ -92,6 +93,14 @@ public void append(Object rowkey, String[] columns, Object[] values, boolean wit append(new Object[] { rowkey }, columns, values, withResult); } + /** + * Put. + */ + @Override + public void put(Object rowkey, String[] columns, Object[] values) { + put(new Object[] { rowkey }, columns, values); + } + /** * Get table name. */ @@ -100,14 +109,14 @@ public String getTableName() { return tableName; } - /** + /* * Set table name. */ public void setTableName(String tableName) { this.tableName = tableName; } - /** + /* * Set atomic operation. */ @Override @@ -115,7 +124,7 @@ public void setAtomicOperation(boolean atomicOperation) { this.atomicOperation = atomicOperation; } - /** + /* * Is atomic operation. */ @Override @@ -123,7 +132,20 @@ public boolean isAtomicOperation() { return atomicOperation; } - /** + @Override + public void setReturnOneResult(boolean returnOneResult) { + this.returnOneResult = returnOneResult; + } + + /* + * Is Return One Result. + */ + @Override + public boolean isReturnOneResult() { + return returnOneResult; + } + + /* * Set entity type. */ @Override @@ -131,7 +153,7 @@ public void setEntityType(ObTableEntityType entityType) { this.entityType = entityType; } - /** + /* * Get entity type. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableQuery.java b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableQuery.java index 8e032e14..b3ea368f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableQuery.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableQuery.java @@ -17,17 +17,22 @@ package com.alipay.oceanbase.rpc.table; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; import com.alipay.oceanbase.rpc.table.api.TableQuery; +import java.util.List; + public abstract class AbstractTableQuery implements TableQuery { private static final String PRIMARY_INDEX_NAME = "PRIMARY"; - protected ObTableEntityType entityType = ObTableEntityType.DYNAMIC; + protected ObTableEntityType entityType = ObTableEntityType.KV; protected long operationTimeout = -1; - /** + /* * Limit. */ @Override @@ -35,7 +40,7 @@ public TableQuery limit(int limit) { return limit(0, limit); } - /** + /* * Primary index. */ @Override @@ -43,7 +48,7 @@ public TableQuery primaryIndex() { return indexName(PRIMARY_INDEX_NAME); } - /** + /* * Add scan range. */ @Override @@ -51,7 +56,7 @@ public TableQuery addScanRange(Object start, Object end) { return addScanRange(new Object[] { start }, true, new Object[] { end }, true); } - /** + /* * Add scan range. */ @Override @@ -59,7 +64,7 @@ public TableQuery addScanRange(Object[] start, Object[] end) { return addScanRange(start, true, end, true); } - /** + /* * Add scan range. */ @Override @@ -67,7 +72,7 @@ public TableQuery addScanRange(Object start, boolean startEquals, Object end, bo return addScanRange(new Object[] { start }, startEquals, new Object[] { end }, endEquals); } - /** + /* * Add scan range starts with. */ @Override @@ -75,7 +80,7 @@ public TableQuery addScanRangeStartsWith(Object start) { return addScanRangeStartsWith(new Object[] { start }, true); } - /** + /* * Add scan range starts with. */ @Override @@ -83,7 +88,7 @@ public TableQuery addScanRangeStartsWith(Object[] start) { return addScanRangeStartsWith(start, true); } - /** + /* * Add scan range ends with. */ @Override @@ -91,7 +96,7 @@ public TableQuery addScanRangeEndsWith(Object end) { return addScanRangeEndsWith(new Object[] { end }, true); } - /** + /* * Add scan range ends with. */ @Override @@ -99,7 +104,7 @@ public TableQuery addScanRangeEndsWith(Object[] end) { return addScanRangeEndsWith(end, true); } - /** + /* * Set entity type. */ @Override @@ -107,7 +112,7 @@ public void setEntityType(ObTableEntityType entityType) { this.entityType = entityType; } - /** + /* * Get entity type. */ @Override @@ -115,7 +120,7 @@ public ObTableEntityType getEntityType() { return entityType; } - /** + /* * Set operation timeout. */ @Override @@ -123,4 +128,37 @@ public TableQuery setOperationTimeout(long operationTimeout) { this.operationTimeout = operationTimeout; return this; } + + /* + * Set filter + */ + @Override + public TableQuery setFilter(ObTableFilter filter) { + if (null == filter) { + throw new IllegalArgumentException("input filter is null"); + } + this.getObTableQuery().setFilterString(filter.toString()); + return this; + } + + @Override + public TableQuery setScanRangeColumns(String... columns) { + this.getObTableQuery().setScanRangeColumns(columns); + return this; + } + + @Override + public List getSelectColumns() { + throw new ObTableException("only query from BatchOperation support getSelectColumns"); + } + + @Override + public Row getRowKey() throws Exception { + throw new ObTableException("only query from BatchOperation support row key"); + } + + @Override + public TableQuery setRowKey(Row row) throws Exception { + throw new ObTableException("only query from BatchOperation support row key"); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableQueryImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableQueryImpl.java index 8fcb13f8..eca0f246 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableQueryImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/AbstractTableQueryImpl.java @@ -26,12 +26,15 @@ import com.alipay.oceanbase.rpc.table.api.TableQuery; import java.util.Arrays; +import java.util.List; public abstract class AbstractTableQueryImpl extends AbstractTableQuery { protected ObTableQuery tableQuery; + // TableEntry key + protected String indexTableName; - /** + /* * Select. */ @Override @@ -40,7 +43,14 @@ public TableQuery select(String... columns) { return this; } - /** + /* + * get select columns' name + */ + public List getSelectColumns() { + return this.tableQuery.getSelectColumns(); + } + + /* * Limit. */ @Override @@ -50,7 +60,7 @@ public TableQuery limit(int limit) { return this; } - /** + /* * Limit. */ @Override @@ -60,7 +70,7 @@ public TableQuery limit(int offset, int limit) { return this; } - /** + /* * Add scan range. */ @Override @@ -85,7 +95,7 @@ public TableQuery addScanRange(Object[] start, boolean startEquals, Object[] end return this; } - /** + /* * Add scan range starts with. */ @Override @@ -104,7 +114,7 @@ public TableQuery addScanRangeStartsWith(Object[] start, boolean startEquals) { return this; } - /** + /* * Add scan range ends with. */ @Override @@ -123,7 +133,7 @@ public TableQuery addScanRangeEndsWith(Object[] end, boolean endEquals) { return this; } - /** + /* * Scan order. */ @Override @@ -132,7 +142,7 @@ public TableQuery scanOrder(boolean forward) { return this; } - /** + /* * Index name. */ @Override @@ -141,7 +151,7 @@ public TableQuery indexName(String indexName) { return this; } - /** + /* * Filter string. */ @Override @@ -150,7 +160,7 @@ public TableQuery filterString(String filterString) { return this; } - /** + /* * Set h table filter. */ @Override @@ -159,7 +169,7 @@ public TableQuery setHTableFilter(ObHTableFilter obHTableFilter) { return this; } - /** + /* * Set batch size. */ @Override @@ -167,4 +177,18 @@ public TableQuery setBatchSize(int batchSize) { this.tableQuery.setBatchSize(batchSize); return this; } + + @Override + public TableQuery setMaxResultSize(long maxResultSize) { + this.tableQuery.setMaxResultSize(maxResultSize); + return this; + } + + public String getIndexTableName() { + return indexTableName; + } + + public void setIndexTableName(String indexTableName) { + this.indexTableName = indexTableName; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ConcurrentTask.java b/src/main/java/com/alipay/oceanbase/rpc/table/ConcurrentTask.java index ddf29739..3b9becb7 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ConcurrentTask.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ConcurrentTask.java @@ -25,7 +25,7 @@ public abstract class ConcurrentTask implements Runnable { private CountDownLatch countDownLatch; private AtomicBoolean stopped; - /** + /* * Run. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ConcurrentTaskExecutor.java b/src/main/java/com/alipay/oceanbase/rpc/table/ConcurrentTaskExecutor.java index fd58ad94..0bcc4f9a 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ConcurrentTaskExecutor.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ConcurrentTaskExecutor.java @@ -32,7 +32,7 @@ public class ConcurrentTaskExecutor { private final List throwableList = Collections .synchronizedList(new LinkedList()); - /** + /* * Concurrent task executor. */ public ConcurrentTaskExecutor(ExecutorService executor, int taskNum) { @@ -41,7 +41,7 @@ public ConcurrentTaskExecutor(ExecutorService executor, int taskNum) { this.executor = executor; } - /** + /* * Execute. */ public void execute(final ConcurrentTask task) { @@ -49,35 +49,35 @@ public void execute(final ConcurrentTask task) { executor.execute(task); } - /** + /* * Is complete. */ public boolean isComplete() { return taskCountDownLatch.getCount() == 0; } - /** + /* * Wait complete. */ public boolean waitComplete(long timeout, TimeUnit unit) throws InterruptedException { return taskCountDownLatch.await(timeout, unit); } - /** + /* * Stop. */ public void stop() { stopped.compareAndSet(false, true); } - /** + /* * Collect exceptions. */ public void collectExceptions(Throwable e) { throwableList.add(e); } - /** + /* * Get throwable list. */ public List getThrowableList() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadBucket.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadBucket.java new file mode 100644 index 00000000..464de222 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadBucket.java @@ -0,0 +1,109 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.table; + +import java.util.ArrayList; +import java.util.List; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +public class ObDirectLoadBucket implements ObSimplePayload { + + private ArrayList rowList = new ArrayList(); + + public ObDirectLoadBucket() { + } + + public boolean isEmpty() { + return rowList.isEmpty(); + } + + public int getRowCount() { + return rowList.size(); + } + + public void addRow(ObObj[] row) { + rowList.add(new ObDirectLoadObjRow(row)); + } + + public void addRow(List row) { + rowList.add(new ObDirectLoadObjRow(row)); + } + + public void addRow(ObDirectLoadObjRow row) { + if (row == null) { + throw new NullPointerException(); + } + rowList.add(row); + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVi32(buf, rowList.size()); + for (int i = 0; i < rowList.size(); ++i) { + rowList.get(i).encode(buf); + } + } + + /** + * Decode. + */ + @Override + public ObDirectLoadBucket decode(ByteBuf buf) { + int count = Serialization.decodeVi32(buf); + for (int i = 0; i < count; ++i) { + ObDirectLoadObjRow row = new ObDirectLoadObjRow(); + row.decode(buf); + rowList.add(row); + } + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + int size = 0; + size += Serialization.getNeedBytes(rowList.size()); + for (int i = 0; i < rowList.size(); ++i) { + size += rowList.get(i).getEncodedSize(); + } + return size; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadObjRow.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadObjRow.java new file mode 100644 index 00000000..34cd4483 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadObjRow.java @@ -0,0 +1,124 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.table; + +import java.util.List; + +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.util.ObByteBuf; +import com.alipay.oceanbase.rpc.util.Serialization; + +import io.netty.buffer.ByteBuf; + +public class ObDirectLoadObjRow implements ObSimplePayload { + + private long SeqNo = 0; + private ObObj[] cells = new ObObj[0]; + + public ObDirectLoadObjRow() { + } + + public ObDirectLoadObjRow(ObObj[] cells) { + setCells(cells); + setSeqNo(0); + } + + public ObDirectLoadObjRow(List cells) { + setCells(cells); + setSeqNo(0); + } + + public ObObj[] getCells() { + return cells; + } + + public void setCells(ObObj[] cells) { + if (cells == null) + throw new NullPointerException(); + this.cells = cells; + } + + public void setCells(List cells) { + if (cells == null) + throw new NullPointerException(); + this.cells = cells.toArray(new ObObj[0]); + } + + public void setSeqNo(long SeqNo) { + this.SeqNo = SeqNo; + } + + public long getSeqNo() { + return SeqNo; + } + + /** + * Encode. + */ + @Override + public byte[] encode() { + int needBytes = getEncodedSize(); + ObByteBuf buf = new ObByteBuf(needBytes); + encode(buf); + return buf.bytes; + } + + /** + * Encode. + */ + @Override + public void encode(ObByteBuf buf) { + Serialization.encodeVi64(buf, SeqNo); + Serialization.encodeVi32(buf, cells.length); + for (int i = 0; i < cells.length; ++i) { + cells[i].encode(buf); + } + } + + /** + * Decode. + */ + @Override + public ObDirectLoadObjRow decode(ByteBuf buf) { + SeqNo = Serialization.decodeVi64(buf); + int count = Serialization.decodeVi32(buf); + cells = new ObObj[count]; + for (int i = 0; i < count; ++i) { + ObObj obj = new ObObj(); + obj.decode(buf); + cells[i] = obj; + } + return this; + } + + /** + * Get encoded size. + */ + @Override + public int getEncodedSize() { + int size = 0; + size += Serialization.getNeedBytes(SeqNo); + size += Serialization.getNeedBytes(cells.length); + for (int i = 0; i < cells.length; ++i) { + size += cells[i].getEncodedSize(); + } + return size; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadParameter.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadParameter.java new file mode 100644 index 00000000..0199b7f6 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObDirectLoadParameter.java @@ -0,0 +1,93 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.table; + +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObLoadDupActionType; + +public class ObDirectLoadParameter { + + private long parallel = 0; + private long maxErrorRowCount = 0; + private ObLoadDupActionType dupAction = ObLoadDupActionType.INVALID_MODE; + private long timeout = 0; + private long heartBeatTimeout = 0; + + public ObDirectLoadParameter() { + } + + public ObDirectLoadParameter(long parallel, long maxErrorRowCount, + ObLoadDupActionType dupAction, long timeout, long heartBeatTimeout) { + this.parallel = parallel; + this.maxErrorRowCount = maxErrorRowCount; + this.dupAction = dupAction; + this.timeout = timeout; + this.heartBeatTimeout = heartBeatTimeout; + } + + public long getParallel() { + return parallel; + } + + public ObDirectLoadParameter setParallel(int parallel) { + this.parallel = parallel; + return this; + } + + public long getMaxErrorRowCount() { + return maxErrorRowCount; + } + + public ObDirectLoadParameter setMaxErrorRowCount(long maxErrorRowCount) { + this.maxErrorRowCount = maxErrorRowCount; + return this; + } + + public ObLoadDupActionType getDupAction() { + return dupAction; + } + + public ObDirectLoadParameter setDupAction(ObLoadDupActionType dupAction) { + this.dupAction = dupAction; + return this; + } + + public long getTimeout() { + return timeout; + } + + public ObDirectLoadParameter setTimeout(long timeout) { + this.timeout = timeout; + return this; + } + + public long getHeartBeatTimeout() { + return heartBeatTimeout; + } + + public ObDirectLoadParameter setHeartBeatTimeout(long heartBeatTimeout) { + this.heartBeatTimeout = heartBeatTimeout; + return this; + } + + public String toString() { + return String.format( + "{parallel:%s, maxErrorRowCount:%d, dupAction:%s, timeout:%d, heartBeatTimeout:%d}", + parallel, maxErrorRowCount, dupAction, timeout, heartBeatTimeout); + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java index 14892028..a184bf05 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTable.java @@ -22,16 +22,12 @@ import com.alipay.oceanbase.rpc.bolt.transport.ObPacketFactory; import com.alipay.oceanbase.rpc.bolt.transport.ObTableConnection; import com.alipay.oceanbase.rpc.bolt.transport.ObTableRemoting; -import com.alipay.oceanbase.rpc.exception.ExceptionUtil; -import com.alipay.oceanbase.rpc.exception.ObTableConnectionStatusException; -import com.alipay.oceanbase.rpc.exception.ObTableException; -import com.alipay.oceanbase.rpc.exception.ObTableServerConnectException; -import com.alipay.oceanbase.rpc.batch.QueryByBatch; +import com.alipay.oceanbase.rpc.checkandmutate.CheckAndInsUp; +import com.alipay.oceanbase.rpc.exception.*; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; +import com.alipay.oceanbase.rpc.mutation.*; import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObITableEntity; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationRequest; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationResult; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; import com.alipay.oceanbase.rpc.table.api.TableBatchOps; import com.alipay.oceanbase.rpc.table.api.TableQuery; import com.alipay.remoting.ConnectionEventHandler; @@ -43,6 +39,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; import static com.alipay.oceanbase.rpc.property.Property.*; @@ -56,16 +53,17 @@ public class ObTable extends AbstractObTable implements Lifecycle { private String userName; private String password; private String database; - private ConnectionFactory connectionFactory; private ObTableRemoting realClient; private ObTableConnectionPool connectionPool; private volatile boolean initialized = false; private volatile boolean closed = false; + private boolean reRouting = false; // only used for init packet factory + private ReentrantLock statusLock = new ReentrantLock(); - /** + /* * Init. */ public void init() throws Exception { @@ -78,12 +76,13 @@ public void init() throws Exception { return; } initProperties(); + init_check(); connectionFactory = ObConnectionFactory .newBuilder() .configWriteBufferWaterMark(getNettyBufferLowWatermark(), getNettyBufferHighWatermark()).build(); connectionFactory.init(new ConnectionEventHandler(new GlobalSwitch())); // Only for monitoring connection status - realClient = new ObTableRemoting(new ObPacketFactory()); + realClient = new ObTableRemoting(new ObPacketFactory(reRouting)); connectionPool = new ObTableConnectionPool(this, obTableConnectionPoolSize); connectionPool.init(); initialized = true; @@ -92,7 +91,7 @@ public void init() throws Exception { } } - /** + /* * Close. */ public void close() { @@ -113,6 +112,21 @@ public void close() { } } + /* + * Init check + */ + private void init_check() throws IllegalArgumentException { + if (obTableConnectionPoolSize <= 0) { + throw new IllegalArgumentException("invalid obTableConnectionPoolSize: " + + obTableConnectionPoolSize); + } else if (ip.isEmpty() || port <= 0) { + throw new IllegalArgumentException("invalid ip or port: " + ip + ":" + port); + } else if (userName.isEmpty() || database.isEmpty()) { + throw new IllegalArgumentException("invalid userName or database: " + userName + ":" + + database); + } + } + private void checkStatus() throws IllegalStateException { if (!initialized) { throw new IllegalStateException(" database [" + database + "] in ip [" + ip @@ -129,36 +143,32 @@ private void checkStatus() throws IllegalStateException { private void initProperties() { obTableConnectTimeout = parseToInt(RPC_CONNECT_TIMEOUT.getKey(), obTableConnectTimeout); + obTableConnectTryTimes = parseToInt(RPC_CONNECT_TRY_TIMES.getKey(), obTableConnectTryTimes); obTableExecuteTimeout = parseToInt(RPC_EXECUTE_TIMEOUT.getKey(), obTableExecuteTimeout); obTableLoginTimeout = parseToInt(RPC_LOGIN_TIMEOUT.getKey(), obTableLoginTimeout); + obTableLoginTryTimes = parseToInt(RPC_LOGIN_TRY_TIMES.getKey(), obTableLoginTryTimes); obTableOperationTimeout = parseToLong(RPC_OPERATION_TIMEOUT.getKey(), obTableOperationTimeout); obTableConnectionPoolSize = parseToInt(SERVER_CONNECTION_POOL_SIZE.getKey(), obTableConnectionPoolSize); - obTableConnectTryTimes = parseToInt(RPC_CONNECT_TRY_TIMES.getKey(), obTableConnectTryTimes); - obTableLoginTryTimes = parseToInt(RPC_LOGIN_TRY_TIMES.getKey(), obTableLoginTryTimes); nettyBufferLowWatermark = parseToInt(NETTY_BUFFER_LOW_WATERMARK.getKey(), nettyBufferLowWatermark); nettyBufferHighWatermark = parseToInt(NETTY_BUFFER_HIGH_WATERMARK.getKey(), nettyBufferHighWatermark); nettyBlockingWaitInterval = parseToInt(NETTY_BLOCKING_WAIT_INTERVAL.getKey(), nettyBlockingWaitInterval); + reRouting = parseToBoolean(SERVER_ENABLE_REROUTING.getKey(), reRouting); } - /** + /* * Query. */ @Override public TableQuery query(String tableName) throws Exception { - return new ObTableQueryImpl(tableName, this); - } - - @Override - public TableQuery queryByBatch(String tableName) throws Exception { - return new QueryByBatch(query(tableName)); + throw new IllegalArgumentException("query using ObTable directly is not supported"); } - /** + /* * Batch. */ @Override @@ -176,40 +186,61 @@ public Map get(String tableName, Object[] rowkeys, String[] colu throws RemotingException, InterruptedException { ObTableOperationResult result = execute(tableName, ObTableOperationType.GET, rowkeys, - columns, null, false, false, true); + columns, null, ObTableOptionFlag.DEFAULT, false, true); ObITableEntity entity = result.getEntity(); return entity.getSimpleProperties(); } /** + * delete. + */ + public Update update(String tableName) { + return new Update(this, tableName); + } + + /* * Update. */ public long update(String tableName, Object[] rowkeys, String[] columns, Object[] values) throws RemotingException, InterruptedException { ObTableOperationResult result = execute(tableName, ObTableOperationType.UPDATE, rowkeys, - columns, values, false, false, true); + columns, values, ObTableOptionFlag.DEFAULT, false, true); return result.getAffectedRows(); } /** + * delete. + */ + public Delete delete(String tableName) { + return new Delete(this, tableName); + } + + /* * Delete. */ public long delete(String tableName, Object[] rowkeys) throws RemotingException, InterruptedException { ObTableOperationResult result = execute(tableName, ObTableOperationType.DEL, rowkeys, null, - null, false, false, true); + null, ObTableOptionFlag.DEFAULT, false, true); return result.getAffectedRows(); } /** * Insert. */ + public Insert insert(String tableName) { + return new Insert(this, tableName); + } + + /* + * Insert. + */ public long insert(String tableName, Object[] rowkeys, String[] columns, Object[] values) throws RemotingException, InterruptedException { ObTableOperationResult result = execute(tableName, ObTableOperationType.INSERT, rowkeys, - columns, values, false, false, true); + columns, values, ObTableOptionFlag.DEFAULT, false, true); return result.getAffectedRows(); } @@ -217,55 +248,106 @@ public long insert(String tableName, Object[] rowkeys, String[] columns, Object[ /** * Replace. */ + public Replace replace(String tableName) { + return new Replace(this, tableName); + } + + /* + * Replace. + */ public long replace(String tableName, Object[] rowkeys, String[] columns, Object[] values) throws RemotingException, InterruptedException { ObTableOperationResult result = execute(tableName, ObTableOperationType.REPLACE, rowkeys, - columns, values, false, false, true); + columns, values, ObTableOptionFlag.DEFAULT, false, true); return result.getAffectedRows(); } /** + * Insert Or Update. + */ + public InsertOrUpdate insertOrUpdate(String tableName) { + return new InsertOrUpdate(this, tableName); + } + + /* * Insert or update. */ public long insertOrUpdate(String tableName, Object[] rowkeys, String[] columns, Object[] values) throws RemotingException, InterruptedException { ObTableOperationResult result = execute(tableName, ObTableOperationType.INSERT_OR_UPDATE, - rowkeys, columns, values, false, false, true); + rowkeys, columns, values, ObTableOptionFlag.DEFAULT, false, true); return result.getAffectedRows(); } + /** + * Put. + */ + public Put put(String tableName) { + return new Put(this, tableName); + } + + /** + * increment. + */ + public Increment increment(String tableName) { + return new Increment(this, tableName); + } + @Override public Map increment(String tableName, Object[] rowkeys, String[] columns, Object[] values, boolean withResult) throws Exception { ObTableOperationResult result = execute(tableName, ObTableOperationType.INCREMENT, rowkeys, - columns, values, false, withResult, true); + columns, values, ObTableOptionFlag.DEFAULT, withResult, true); ObITableEntity entity = result.getEntity(); return entity.getSimpleProperties(); } + /** + * append. + */ + public Append append(String tableName) { + return new Append(this, tableName); + } + @Override public Map append(String tableName, Object[] rowkeys, String[] columns, Object[] values, boolean withResult) throws Exception { ObTableOperationResult result = execute(tableName, ObTableOperationType.APPEND, rowkeys, - columns, values, false, withResult, true); + columns, values, ObTableOptionFlag.DEFAULT, withResult, true); ObITableEntity entity = result.getEntity(); return entity.getSimpleProperties(); } /** + * batch mutation. + */ + public BatchOperation batchOperation(String tableName) { + return new BatchOperation(this, tableName); + } + + /** + * Insert. + */ + public CheckAndInsUp checkAndInsUp(String tableName, ObTableFilter filter, + InsertOrUpdate insUp, boolean checkExists) { + return new CheckAndInsUp(this, tableName, filter, insUp, checkExists); + } + + /* * Execute. */ public ObTableOperationResult execute(String tableName, ObTableOperationType type, Object[] rowkeys, String[] columns, Object[] values, - boolean returningRowKey, boolean returningAffectedEntity, + ObTableOptionFlag optionFlag, + boolean returningAffectedEntity, boolean returningAffectedRows) throws RemotingException, InterruptedException { checkStatus(); ObTableOperationRequest request = ObTableOperationRequest.getInstance(tableName, type, rowkeys, columns, values, obTableOperationTimeout); - request.setReturningRowKey(returningRowKey); + request.setOptionFlag(optionFlag); request.setReturningAffectedEntity(returningAffectedEntity); request.setReturningAffectedRows(returningAffectedRows); ObPayload result = execute(request); @@ -273,14 +355,15 @@ public ObTableOperationResult execute(String tableName, ObTableOperationType typ return (ObTableOperationResult) result; } - /** + /* * Execute. */ public ObPayload execute(final ObPayload request) throws RemotingException, InterruptedException { - ObTableConnection connection = getConnection(); + ObTableConnection connection = null; try { + connection = getConnection(); // check connection is available, if not available, reconnect it connection.checkStatus(); } catch (ConnectException ex) { @@ -291,8 +374,67 @@ public ObPayload execute(final ObPayload request) throws RemotingException, } catch (Exception ex) { throw new ObTableConnectionStatusException("check status failed", ex); } + return executeWithReconnect(connection, request); + } + + private ObPayload executeWithReconnect(ObTableConnection connection, final ObPayload request) + throws RemotingException, + InterruptedException { + boolean needReconnect = false; + int retryTimes = 0; + ObPayload payload = null; + do { + retryTimes++; + try { + if (needReconnect) { + String msg = String + .format( + "Receive error: tenant not in server and reconnect it, ip:{}, port:{}, tenant id:{}, retryTimes: {}", + connection.getObTable().getIp(), connection.getObTable().getPort(), + connection.getTenantId(), retryTimes); + connection.reConnectAndLogin(msg); + needReconnect = false; + } + payload = realClient.invokeSync(connection, request, obTableExecuteTimeout); + } catch (ObTableException ex) { + if (ex instanceof ObTableTenantNotInServerException && retryTimes < 2) { + needReconnect = true; + } else { + throw ex; + } + } + } while (needReconnect && retryTimes < 2); + return payload; + } - return realClient.invokeSync(connection, request, obTableExecuteTimeout); + /* + * Execute with certain connection + * If connection is null, this method will replace the connection with random connection + * If connection is not null, this method will use that connection to execute + */ + public ObPayload executeWithConnection(final ObPayload request, + AtomicReference connectionRef) + throws RemotingException, + InterruptedException { + ObTableConnection connection; + try { + if (connectionRef.get() == null) { + // Set a connection into ref if connection is null + connection = getConnection(); + connectionRef.set(connection); + } + connection = connectionRef.get(); + // Check connection is available, if not available, reconnect it + connection.checkStatus(); + } catch (ConnectException ex) { + // Cannot connect to ob server, need refresh table location + throw new ObTableServerConnectException(ex); + } catch (ObTableServerConnectException ex) { + throw ex; + } catch (Exception ex) { + throw new ObTableConnectionStatusException("check status failed", ex); + } + return executeWithReconnect(connection, request); } private void checkObTableOperationResult(String ip, int port, Object result) { @@ -309,126 +451,138 @@ private void checkObTableOperationResult(String ip, int port, Object result) { ((ObTableOperationResult) result).setExecuteHost(ip); ((ObTableOperationResult) result).setExecutePort(port); ExceptionUtil.throwObTableException(ip, port, obTableOperationResult.getSequence(), - obTableOperationResult.getUniqueId(), obTableOperationResult.getHeader().getErrno()); + obTableOperationResult.getUniqueId(), obTableOperationResult.getHeader().getErrno(), + obTableOperationResult.getHeader().getErrMsg()); } - /** + /* * Get ip. */ public String getIp() { return ip; } - /** + /* * Set ip. */ public void setIp(String ip) { this.ip = ip; } - /** + /* * Get port. */ public int getPort() { return port; } - /** + /* * Set port. */ public void setPort(int port) { this.port = port; } - /** + /* * Get tenant name. */ public String getTenantName() { return tenantName; } - /** + /* * Set tenant name. */ public void setTenantName(String tenantName) { this.tenantName = tenantName; } - /** + /* * Get user name. */ public String getUserName() { return userName; } - /** + /* * Set user name. */ public void setUserName(String userName) { this.userName = userName; } - /** + /* * Get password. */ public String getPassword() { return password; } - /** + /* * Set password. */ public void setPassword(String password) { this.password = password; } - /** + /* * Get database. */ public String getDatabase() { return database; } - /** + /* * Set database. */ public void setDatabase(String database) { this.database = database; } - /** + /* * Get connection factory. */ public ConnectionFactory getConnectionFactory() { return connectionFactory; } - /** + /* * Set connection factory. */ public void setConnectionFactory(ConnectionFactory connectionFactory) { this.connectionFactory = connectionFactory; } - /** + /* * Get real client. */ public ObTableRemoting getRealClient() { return realClient; } - /** + /* * Set real client. */ public void setRealClient(ObTableRemoting realClient) { this.realClient = realClient; } - /** + /* * Get connection. */ - public ObTableConnection getConnection() { - return connectionPool.getConnection(); + public ObTableConnection getConnection() throws Exception { + ObTableConnection conn = connectionPool.getConnection(); + int count = 0; + while (conn.getConnection() != null + && (conn.getCredential() == null || conn.getCredential().length() == 0) + && count < obTableConnectionPoolSize) { + conn = connectionPool.getConnection(); + count++; + } + if (count == obTableConnectionPoolSize) { + throw new ObTableException("all connection's credential is null"); + } + return conn; } public static class Builder { @@ -443,7 +597,7 @@ public static class Builder { private Properties properties = new Properties(); - /** + /* * Builder. */ public Builder(String ip, int port) { @@ -451,7 +605,7 @@ public Builder(String ip, int port) { this.port = port; } - /** + /* * Set login info. */ public Builder setLoginInfo(String tenantName, String userName, String password, @@ -463,7 +617,7 @@ public Builder setLoginInfo(String tenantName, String userName, String password, return this; } - /** + /* * Add propery. */ public Builder addPropery(String key, String value) { @@ -471,7 +625,7 @@ public Builder addPropery(String key, String value) { return this; } - /** + /* * Set properties. */ public Builder setProperties(Properties properties) { @@ -479,7 +633,7 @@ public Builder setProperties(Properties properties) { return this; } - /** + /* * Build. */ public ObTable build() throws Exception { @@ -499,15 +653,15 @@ public ObTable build() throws Exception { } - /** + /* * A simple pool for ObTableConnection with fix size. Redesign it when we needs more. * The scheduling policy is round-robin. It's also simple but enough currently. Now, we promise sequential * consistency, while each thread call invokeSync for data access, ensuring the sequential consistency. *

* Thread safety: - * （1） init and close require external synchronization or locking, which should be called only once. - * （2） getConnection from the pool is synchronized, thus thread-safe, . - * （3） ObTableConnection is shared and thread-safe, granted by underlying library. + * (1) init and close require external synchronization or locking, which should be called only once. + * (2) getConnection from the pool is synchronized, thus thread-safe, . + * (3) ObTableConnection is shared and thread-safe, granted by underlying library. */ private static class ObTableConnectionPool { private final int obTableConnectionPoolSize; @@ -516,7 +670,7 @@ private static class ObTableConnectionPool { // round-robin scheduling private AtomicLong turn = new AtomicLong(0); - /** + /* * Ob table connection pool. */ public ObTableConnectionPool(ObTable obTable, int connectionPoolSize) { @@ -525,7 +679,7 @@ public ObTableConnectionPool(ObTable obTable, int connectionPoolSize) { connectionPool = new ObTableConnection[connectionPoolSize]; } - /** + /* * Init. */ public void init() throws Exception { @@ -535,7 +689,7 @@ public void init() throws Exception { } } - /** + /* * Get connection. */ public ObTableConnection getConnection() { @@ -543,7 +697,7 @@ public ObTableConnection getConnection() { return connectionPool[round]; } - /** + /* * Close. */ public void close() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java index f05ee553..7e984422 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableBatchOpsImpl.java @@ -19,6 +19,8 @@ import com.alipay.oceanbase.rpc.exception.ExceptionUtil; import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.mutation.result.*; import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; import com.alipay.remoting.exception.RemotingException; @@ -33,7 +35,7 @@ public class ObTableBatchOpsImpl extends AbstractTableBatchOps { private ObTable obTable; - /** + /* * Ob table batch ops impl. */ public ObTableBatchOpsImpl(String tableName, ObTable obTable) { @@ -48,7 +50,7 @@ public ObTableBatchOpsImpl(String tableName, ObTable obTable) { request.setReturningAffectedRows(true); } - /** + /* * Get. */ @Override @@ -56,7 +58,7 @@ public void get(Object[] rowkeys, String[] columns) { addObTableOperation(ObTableOperationType.GET, rowkeys, columns, null); } - /** + /* * Update. */ @Override @@ -64,7 +66,7 @@ public void update(Object[] rowkeys, String[] columns, Object[] values) { addObTableOperation(ObTableOperationType.UPDATE, rowkeys, columns, values); } - /** + /* * Delete. */ @Override @@ -72,7 +74,7 @@ public void delete(Object[] rowkeys) { addObTableOperation(ObTableOperationType.DEL, rowkeys, null, null); } - /** + /* * Insert. */ @Override @@ -80,7 +82,7 @@ public void insert(Object[] rowkeys, String[] columns, Object[] values) { addObTableOperation(ObTableOperationType.INSERT, rowkeys, columns, values); } - /** + /* * Replace. */ @Override @@ -88,7 +90,7 @@ public void replace(Object[] rowkeys, String[] columns, Object[] values) { addObTableOperation(ObTableOperationType.REPLACE, rowkeys, columns, values); } - /** + /* * Insert or update. */ @Override @@ -96,7 +98,7 @@ public void insertOrUpdate(Object[] rowkeys, String[] columns, Object[] values) addObTableOperation(ObTableOperationType.INSERT_OR_UPDATE, rowkeys, columns, values); } - /** + /* * Increment. */ @Override @@ -105,7 +107,7 @@ public void increment(Object[] rowkeys, String[] columns, Object[] values, boole addObTableOperation(ObTableOperationType.INCREMENT, rowkeys, columns, values); } - /** + /* * Append. */ @Override @@ -114,7 +116,15 @@ public void append(Object[] rowkeys, String[] columns, Object[] values, boolean addObTableOperation(ObTableOperationType.APPEND, rowkeys, columns, values); } - /** + /* + * Put. + */ + @Override + public void put(Object[] rowkeys, String[] columns, Object[] values) { + addObTableOperation(ObTableOperationType.PUT, rowkeys, columns, values); + } + + /* * Add ob table operation. */ public void addObTableOperation(ObTableOperationType type, Object[] rowkeys, String[] columns, @@ -124,12 +134,13 @@ public void addObTableOperation(ObTableOperationType type, Object[] rowkeys, Str operations.addTableOperation(instance); } - /** + /* * Execute. */ public List execute() throws RemotingException, InterruptedException { request.setBatchOperationAsAtomic(isAtomicOperation()); + request.setBatchOpReturnOneResult(isReturnOneResult()); Object result = obTable.execute(request); checkObTableOperationResult(result); @@ -137,8 +148,8 @@ public List execute() throws RemotingException, InterruptedException { List realResults = obTableOperationResult.getResults(); List results = new ArrayList(realResults.size()); for (ObTableOperationResult realResult : realResults) { - ResultCodes resultCodes = ResultCodes.valueOf(realResult.getHeader().getErrno()); - if (resultCodes == ResultCodes.OB_SUCCESS) { + int errCode = realResult.getHeader().getErrno(); + if (errCode == ResultCodes.OB_SUCCESS.errorCode) { switch (realResult.getOperationType()) { case GET: case INCREMENT: @@ -150,14 +161,55 @@ public List execute() throws RemotingException, InterruptedException { } } else { results.add(ExceptionUtil.convertToObTableException(obTable.getIp(), - obTable.getPort(), realResult.getSequence(), realResult.getUniqueId(), - resultCodes)); + obTable.getPort(), realResult.getSequence(), realResult.getUniqueId(), errCode, + realResult.getHeader().getErrMsg())); + } + } + return results; + } + + /* + * Execute with result + */ + public List executeWithResult() throws Exception { + + request.setBatchOperationAsAtomic(isAtomicOperation()); + request.setBatchOpReturnOneResult(isReturnOneResult()); + Object result = obTable.execute(request); + checkObTableOperationResult(result); + + ObTableBatchOperationResult obTableOperationResult = (ObTableBatchOperationResult) result; + List realResults = obTableOperationResult.getResults(); + List results = new ArrayList(realResults.size()); + for (ObTableOperationResult realResult : realResults) { + int errCode = realResult.getHeader().getErrno(); + if (errCode == ResultCodes.OB_SUCCESS.errorCode) { + switch (realResult.getOperationType()) { + case GET: + throw new ObTableException("Get is not a mutation"); + case INSERT: + case DEL: + case UPDATE: + case INSERT_OR_UPDATE: + case REPLACE: + case INCREMENT: + case APPEND: + results.add(new MutationResult(realResult)); + break; + default: + throw new ObTableException("unknown operation type " + + realResult.getOperationType()); + } + } else { + results.add(ExceptionUtil.convertToObTableException(obTable.getIp(), + obTable.getPort(), realResult.getSequence(), realResult.getUniqueId(), errCode, + realResult.getHeader().getErrMsg())); } } return results; } - /** + /* * clear batch operations */ public void clear() { @@ -178,14 +230,14 @@ private void checkObTableOperationResult(Object result) { // ExceptionUtil.throwObTableException(obTableOperationResult.getHeader().getErrno()); } - /** + /* * Reset ob table. */ public void resetObTable(ObTable obTable) { this.obTable = obTable; } - /** + /* * Get ob table batch operation. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java index aae786e2..f56e7b38 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java @@ -21,10 +21,12 @@ import com.alipay.oceanbase.rpc.exception.*; import com.alipay.oceanbase.rpc.location.model.ObServerRoute; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; +import com.alipay.oceanbase.rpc.mutation.result.*; import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; import com.alipay.oceanbase.rpc.threadlocal.ThreadLocalMap; +import com.alipay.oceanbase.rpc.util.MonitorUtil; import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; import org.slf4j.Logger; @@ -32,7 +34,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; +import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.*; +import static com.alipay.oceanbase.rpc.util.TraceUtil.formatTraceMessage; public class ObTableClientBatchOpsImpl extends AbstractTableBatchOps { @@ -45,7 +48,7 @@ public class ObTableClientBatchOpsImpl extends AbstractTableBatchOps { private boolean returningAffectedEntity = false; private ObTableBatchOperation batchOperation; - /** + /* * Ob table client batch ops impl. */ public ObTableClientBatchOpsImpl(String tableName, ObTableClient obTableClient) { @@ -54,7 +57,7 @@ public ObTableClientBatchOpsImpl(String tableName, ObTableClient obTableClient) this.batchOperation = new ObTableBatchOperation(); } - /** + /* * Ob table client batch ops impl. */ public ObTableClientBatchOpsImpl(String tableName, ObTableBatchOperation batchOperation, @@ -64,7 +67,7 @@ public ObTableClientBatchOpsImpl(String tableName, ObTableBatchOperation batchOp this.batchOperation = batchOperation; } - /** + /* * Get ob table batch operation. */ @Override @@ -72,7 +75,7 @@ public ObTableBatchOperation getObTableBatchOperation() { return batchOperation; } - /** + /* * Get. */ @Override @@ -80,7 +83,7 @@ public void get(Object[] rowkeys, String[] columns) { addObTableClientOperation(ObTableOperationType.GET, rowkeys, columns, null); } - /** + /* * Update. */ @Override @@ -88,7 +91,7 @@ public void update(Object[] rowkeys, String[] columns, Object[] values) { addObTableClientOperation(ObTableOperationType.UPDATE, rowkeys, columns, values); } - /** + /* * Delete. */ @Override @@ -96,7 +99,7 @@ public void delete(Object[] rowkeys) { addObTableClientOperation(ObTableOperationType.DEL, rowkeys, null, null); } - /** + /* * Insert. */ @Override @@ -104,7 +107,7 @@ public void insert(Object[] rowkeys, String[] columns, Object[] values) { addObTableClientOperation(ObTableOperationType.INSERT, rowkeys, columns, values); } - /** + /* * Replace. */ @Override @@ -112,7 +115,7 @@ public void replace(Object[] rowkeys, String[] columns, Object[] values) { addObTableClientOperation(ObTableOperationType.REPLACE, rowkeys, columns, values); } - /** + /* * Insert or update. */ @Override @@ -120,7 +123,7 @@ public void insertOrUpdate(Object[] rowkeys, String[] columns, Object[] values) addObTableClientOperation(ObTableOperationType.INSERT_OR_UPDATE, rowkeys, columns, values); } - /** + /* * Increment. */ @Override @@ -129,7 +132,7 @@ public void increment(Object[] rowkeys, String[] columns, Object[] values, boole addObTableClientOperation(ObTableOperationType.INCREMENT, rowkeys, columns, values); } - /** + /* * Append. */ @Override @@ -138,21 +141,29 @@ public void append(Object[] rowkeys, String[] columns, Object[] values, boolean addObTableClientOperation(ObTableOperationType.APPEND, rowkeys, columns, values); } + /* + * Put. + */ + @Override + public void put(Object[] rowkeys, String[] columns, Object[] values) { + addObTableClientOperation(ObTableOperationType.PUT, rowkeys, columns, values); + } + private void addObTableClientOperation(ObTableOperationType type, Object[] rowkeys, String[] columns, Object[] values) { ObTableOperation instance = ObTableOperation.getInstance(type, rowkeys, columns, values); batchOperation.addTableOperation((instance)); } - /** + /* * Execute. */ public List execute() throws Exception { // consistent can not be sure List results = new ArrayList(batchOperation.getTableOperations().size()); for (ObTableOperationResult result : executeInternal().getResults()) { - ResultCodes resultCodes = ResultCodes.valueOf(result.getHeader().getErrno()); - if (resultCodes == ResultCodes.OB_SUCCESS) { + int errCode = result.getHeader().getErrno(); + if (errCode == ResultCodes.OB_SUCCESS.errorCode) { switch (result.getOperationType()) { case GET: case INCREMENT: @@ -164,18 +175,65 @@ public List execute() throws Exception { } } else { results.add(ExceptionUtil.convertToObTableException(result.getExecuteHost(), - result.getExecutePort(), result.getSequence(), result.getUniqueId(), - resultCodes)); + result.getExecutePort(), result.getSequence(), result.getUniqueId(), errCode, + result.getHeader().getErrMsg())); + } + } + return results; + } + + /* + * Execute with result + */ + public List executeWithResult() throws Exception { + // consistent can not be sure + List results = new ArrayList(batchOperation.getTableOperations().size()); + for (ObTableOperationResult result : executeInternal().getResults()) { + int errCode = result.getHeader().getErrno(); + if (errCode == ResultCodes.OB_SUCCESS.errorCode) { + switch (result.getOperationType()) { + case GET: + case INSERT: + case DEL: + case UPDATE: + case INSERT_OR_UPDATE: + case REPLACE: + case INCREMENT: + case APPEND: + case PUT: + results.add(new MutationResult(result)); + break; + default: + throw new ObTableException("unknown operation type " + + result.getOperationType()); + } + } else { + results.add(ExceptionUtil.convertToObTableException(result.getExecuteHost(), + result.getExecutePort(), result.getSequence(), result.getUniqueId(), errCode, + result.getHeader().getErrMsg())); } } return results; } - public Map>>> partitionPrepare() - throws Exception { + public Map>>> partitionPrepare() + throws Exception { // consistent can not be sure List operations = batchOperation.getTableOperations(); - Map>>> partitionOperationsMap = new HashMap>>>(); + Map>>> partitionOperationsMap = new HashMap>>>(); + + if (obTableClient.isOdpMode()) { + ObPair>> obTableOperations = new ObPair>>( + new ObTableParam(obTableClient.getOdpTable()), + new ArrayList>()); + for (int i = 0; i < operations.size(); i++) { + ObTableOperation operation = operations.get(i); + obTableOperations.getRight().add( + new ObPair(i, operation)); + } + partitionOperationsMap.put(0L, obTableOperations); + return partitionOperationsMap; + } for (int i = 0; i < operations.size(); i++) { ObTableOperation operation = operations.get(i); @@ -185,36 +243,33 @@ public Map>>> parti for (int j = 0; j < rowKeySize; j++) { rowKey[j] = rowKeyObject.getObj(j).getValue(); } - - ObPair tableObPair = obTableClient.getTable(tableName, rowKey, false, - false, obTableClient.getRoute(batchOperation.isReadOnly())); - ObPair>> obTableOperations = partitionOperationsMap + ObPair tableObPair = obTableClient.getTable( + tableName, rowKey, false, false, + obTableClient.getRoute(batchOperation.isReadOnly())); + ObPair>> obTableOperations = partitionOperationsMap .get(tableObPair.getLeft()); if (obTableOperations == null) { - obTableOperations = new ObPair>>( + obTableOperations = new ObPair>>( tableObPair.getRight(), new ArrayList>()); partitionOperationsMap.put(tableObPair.getLeft(), obTableOperations); } obTableOperations.getRight().add(new ObPair(i, operation)); } - if (atomicOperation) { - if (partitionOperationsMap.size() > 1) { - throw new ObTablePartitionConsistentException( - "require atomic operation but found across partition may cause consistent problem "); - } - } return partitionOperationsMap; } - /** + /* * Partition execute. */ public void partitionExecute(ObTableOperationResult[] results, - Map.Entry>>> partitionOperation) - throws Exception { - Long partId = partitionOperation.getKey(); - ObTable subObTable = partitionOperation.getValue().getLeft(); + Map.Entry>>> partitionOperation) + throws Exception { + ObTableParam tableParam = partitionOperation.getValue().getLeft(); + long tableId = tableParam.getTableId(); + long partId = tableParam.getPartitionId(); + long originPartId = tableParam.getPartId(); + ObTable subObTable = tableParam.getObTable(); List> subOperationWithIndexList = partitionOperation .getValue().getRight(); @@ -230,6 +285,7 @@ public void partitionExecute(ObTableOperationResult[] results, subRequest.setTableName(tableName); subRequest.setReturningAffectedEntity(returningAffectedEntity); subRequest.setReturningAffectedRows(true); + subRequest.setTableId(tableId); subRequest.setPartitionId(partId); subRequest.setEntityType(entityType); subRequest.setTimeout(subObTable.getObTableOperationTimeout()); @@ -238,9 +294,11 @@ public void partitionExecute(ObTableOperationResult[] results, .toObTableConsistencyLevel()); } subRequest.setBatchOperationAsAtomic(isAtomicOperation()); + subRequest.setBatchOpReturnOneResult(isReturnOneResult()); ObTableBatchOperationResult subObTableBatchOperationResult; boolean needRefreshTableEntry = false; + boolean needFetchAllRouteInfo = false; int tryTimes = 0; long startExecute = System.currentTimeMillis(); Set failedServerList = null; @@ -251,6 +309,11 @@ public void partitionExecute(ObTableOperationResult[] results, long currentExecute = System.currentTimeMillis(); long costMillis = currentExecute - startExecute; if (costMillis > obTableClient.getRuntimeMaxWait()) { + logger.error( + "tablename:{} partition id:{} it has tried " + tryTimes + + " times and it has waited " + costMillis + + "/ms which exceeds response timeout " + + obTableClient.getRuntimeMaxWait() + "/ms", tableName, partId); throw new ObTableTimeoutExcetion("it has tried " + tryTimes + " times and it has waited " + costMillis + "/ms which exceeds response timeout " @@ -258,46 +321,73 @@ public void partitionExecute(ObTableOperationResult[] results, } tryTimes++; try { - // 重试时重新 getTable - if (tryTimes > 1) { - if (route == null) { - route = obTableClient.getRoute(batchOperation.isReadOnly()); - } - if (failedServerList != null) { - route.setBlackList(failedServerList); + if (obTableClient.isOdpMode()) { + subObTable = obTableClient.getOdpTable(); + } else { + // getTable() when we need retry + // we should use partIdx to get table + if (tryTimes > 1) { + if (route == null) { + route = obTableClient.getRoute(batchOperation.isReadOnly()); + } + if (failedServerList != null) { + route.setBlackList(failedServerList); + } + ObTableParam newParam = obTableClient.getTableWithPartId(tableName, + originPartId, needRefreshTableEntry, + obTableClient.isTableEntryRefreshIntervalWait(), needFetchAllRouteInfo, route).getRight(); + + subObTable = newParam.getObTable(); + subRequest.setPartitionId(newParam.getPartitionId()); } - subObTable = obTableClient.getTable(tableName, partId, needRefreshTableEntry, - obTableClient.isTableEntryRefreshIntervalWait(), route).getRight(); } subObTableBatchOperationResult = (ObTableBatchOperationResult) subObTable .execute(subRequest); obTableClient.resetExecuteContinuousFailureCount(tableName); break; - } catch (ObTableReplicaNotReadableException ex) { - if ((tryTimes - 1) < obTableClient.getRuntimeRetryTimes()) { - logger.warn("retry when replica not readable: {}", ex.getMessage()); - if (failedServerList == null) { - failedServerList = new HashSet(); - } - failedServerList.add(subObTable.getIp()); - } else { - logger.warn("exhaust retry when replica not readable: {}", ex.getMessage()); - throw ex; - } } catch (Exception ex) { - if (ex instanceof ObTableException - && ((ObTableException) ex).isNeedRefreshTableEntry()) { + if (obTableClient.isOdpMode()) { + if ((tryTimes - 1) < obTableClient.getRuntimeRetryTimes()) { + logger + .warn( + "batch ops execute while meet Exception, tablename:{}, errorCode: {} , errorMsg: {}, try times {}", + tableName, ((ObTableException) ex).getErrorCode(), ex.getMessage(), + tryTimes); + } else { + throw ex; + } + } else if (ex instanceof ObTableReplicaNotReadableException) { + if ((tryTimes - 1) < obTableClient.getRuntimeRetryTimes()) { + logger.warn( + "tablename:{} partition id:{} retry when replica not readable: {}", + tableName, partId, ex.getMessage()); + if (failedServerList == null) { + failedServerList = new HashSet(); + } + failedServerList.add(subObTable.getIp()); + } else { + logger.warn("exhaust retry when replica not readable: {}", ex.getMessage()); + throw ex; + } + } else if (ex instanceof ObTableException + && ((ObTableException) ex).isNeedRefreshTableEntry()) { needRefreshTableEntry = true; logger .warn( - "batch ops refresh table while meet ObTableMasterChangeException, errorCode: {}", - ((ObTableException) ex).getErrorCode()); + "tablename:{} partition id:{} batch ops refresh table while meet ObTableMasterChangeException, errorCode: {}", + tableName, partId, ((ObTableException) ex).getErrorCode(), ex); if (obTableClient.isRetryOnChangeMasterTimes() && (tryTimes - 1) < obTableClient.getRuntimeRetryTimes()) { logger .warn( - "batch ops retry while meet ObTableMasterChangeException, errorCode: {} , retry times {}", - ((ObTableException) ex).getErrorCode(), tryTimes); + "tablename:{} partition id:{} batch ops retry while meet ObTableMasterChangeException, errorCode: {} , retry times {}", + tableName, partId, ((ObTableException) ex).getErrorCode(), + tryTimes, ex); + if (ex instanceof ObTableNeedFetchAllException) { + needFetchAllRouteInfo = true; + // reset failure count while fetch all route info + obTableClient.resetExecuteContinuousFailureCount(tableName); + } } else { obTableClient.calculateContinuousFailure(tableName, ex.getMessage()); throw ex; @@ -310,7 +400,13 @@ public void partitionExecute(ObTableOperationResult[] results, Thread.sleep(obTableClient.getRuntimeRetryInterval()); } + long endExecute = System.currentTimeMillis(); + if (subObTableBatchOperationResult == null) { + RUNTIME + .error( + "tablename:{} partition id:{} check batch operation result error: client get unexpected NULL result", + tableName, partId); throw new ObTableUnexpectedException( "check batch operation result error: client get unexpected NULL result"); } @@ -318,68 +414,95 @@ public void partitionExecute(ObTableOperationResult[] results, List subObTableOperationResults = subObTableBatchOperationResult .getResults(); - if (subObTableOperationResults.size() < subOperations.getTableOperations().size()) { - // only one result when it across failed - // only one result when hkv puts - if (subObTableOperationResults.size() == 1) { - ObTableOperationResult subObTableOperationResult = subObTableOperationResults - .get(0); + if (returnOneResult) { + ObTableOperationResult subObTableOperationResult = subObTableOperationResults.get(0); + if (results[0] == null) { + results[0] = new ObTableOperationResult(); subObTableOperationResult.setExecuteHost(subObTable.getIp()); subObTableOperationResult.setExecutePort(subObTable.getPort()); - for (ObPair aSubOperationWithIndexList : subOperationWithIndexList) { - results[aSubOperationWithIndexList.getLeft()] = subObTableOperationResult; - } + results[0] = subObTableOperationResult; } else { - // unexpected result found - throw new IllegalArgumentException( - "check batch operation result size error: operation size [" - + subOperations.getTableOperations().size() + "] result size [" - + subObTableOperationResults.size() + "]"); + results[0].setAffectedRows(results[0].getAffectedRows() + + subObTableOperationResult.getAffectedRows()); } } else { - if (subOperationWithIndexList.size() != subObTableOperationResults.size()) { - throw new ObTableUnexpectedException("check batch result error: partition " - + partId + " expect result size " - + subOperationWithIndexList.size() - + " actual result size " - + subObTableOperationResults.size()); - } - for (int i = 0; i < subOperationWithIndexList.size(); i++) { - ObTableOperationResult subObTableOperationResult = subObTableOperationResults - .get(i); - subObTableOperationResult.setExecuteHost(subObTable.getIp()); - subObTableOperationResult.setExecutePort(subObTable.getPort()); - results[subOperationWithIndexList.get(i).getLeft()] = subObTableOperationResult; + if (subObTableOperationResults.size() < subOperations.getTableOperations().size()) { + // only one result when it across failed + // only one result when hkv puts + if (subObTableOperationResults.size() == 1) { + ObTableOperationResult subObTableOperationResult = subObTableOperationResults + .get(0); + subObTableOperationResult.setExecuteHost(subObTable.getIp()); + subObTableOperationResult.setExecutePort(subObTable.getPort()); + for (ObPair aSubOperationWithIndexList : subOperationWithIndexList) { + results[aSubOperationWithIndexList.getLeft()] = subObTableOperationResult; + } + } else { + // unexpected result found + throw new IllegalArgumentException( + "check batch operation result size error: operation size [" + + subOperations.getTableOperations().size() + "] result size [" + + subObTableOperationResults.size() + "]"); + } + } else { + if (subOperationWithIndexList.size() != subObTableOperationResults.size()) { + throw new ObTableUnexpectedException("check batch result error: partition " + + partId + " expect result size " + + subOperationWithIndexList.size() + + " actual result size " + + subObTableOperationResults.size()); + } + for (int i = 0; i < subOperationWithIndexList.size(); i++) { + ObTableOperationResult subObTableOperationResult = subObTableOperationResults + .get(i); + subObTableOperationResult.setExecuteHost(subObTable.getIp()); + subObTableOperationResult.setExecutePort(subObTable.getPort()); + results[subOperationWithIndexList.get(i).getLeft()] = subObTableOperationResult; + } } } + String endpoint = subObTable.getIp() + ":" + subObTable.getPort(); + MonitorUtil.info(subRequest, subObTable.getDatabase(), tableName, + "BATCH-partitionExecute-", endpoint, subOperations, partId, + subObTableOperationResults.size(), endExecute - startExecute, + obTableClient.getslowQueryMonitorThreshold()); } - ; - - /** + /* * Execute internal. */ public ObTableBatchOperationResult executeInternal() throws Exception { + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + long start = System.currentTimeMillis(); List operations = batchOperation.getTableOperations(); - final ObTableOperationResult[] obTableOperationResults = new ObTableOperationResult[operations - .size()]; - Map>>> partitions = partitionPrepare(); + ObTableOperationResult[] obTableOperationResults = null; + if (returnOneResult) { + obTableOperationResults = new ObTableOperationResult[1]; + } else { + obTableOperationResults = new ObTableOperationResult[operations.size()]; + } + + Map>>> partitions = partitionPrepare(); + long getTableTime = System.currentTimeMillis(); final Map context = ThreadLocalMap.getContextMap(); if (executorService != null && !executorService.isShutdown() && partitions.size() > 1) { final ConcurrentTaskExecutor executor = new ConcurrentTaskExecutor(executorService, partitions.size()); - for (final Map.Entry>>> entry : partitions + for (final Map.Entry>>> entry : partitions .entrySet()) { + ObTableOperationResult[] finalObTableOperationResults = obTableOperationResults; executor.execute(new ConcurrentTask() { - /** + /* * Do task. */ @Override public void doTask() { try { ThreadLocalMap.transmitContextMap(context); - partitionExecute(obTableOperationResults, entry); + partitionExecute(finalObTableOperationResults, entry); } catch (Exception e) { logger.error(LCD.convert("01-00026"), e); executor.collectExceptions(e); @@ -427,7 +550,7 @@ public void doTask() { } } else { - for (final Map.Entry>>> entry : partitionPrepare() + for (final Map.Entry>>> entry : partitions .entrySet()) { partitionExecute(obTableOperationResults, entry); } @@ -437,20 +560,34 @@ public void doTask() { for (ObTableOperationResult obTableOperationResult : obTableOperationResults) { batchOperationResult.addResult(obTableOperationResult); } + + MonitorUtil.info(batchOperationResult, obTableClient.getDatabase(), tableName, "BATCH", "", + obTableOperationResults.length, getTableTime - start, System.currentTimeMillis() + - getTableTime, + obTableClient.getslowQueryMonitorThreshold()); + return batchOperationResult; } - /** + /* * clear batch operations1 */ public void clear() { batchOperation = new ObTableBatchOperation(); } - /** + /* * Set executor service. */ public void setExecutorService(ExecutorService executorService) { this.executorService = executorService; } + + public boolean isReturningAffectedEntity() { + return returningAffectedEntity; + } + + public void setReturningAffectedEntity(boolean returningAffectedEntity) { + this.returningAffectedEntity = returningAffectedEntity; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java new file mode 100644 index 00000000..7c38d8d3 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java @@ -0,0 +1,703 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.table; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.checkandmutate.CheckAndInsUp; +import com.alipay.oceanbase.rpc.exception.*; +import com.alipay.oceanbase.rpc.location.model.ObServerRoute; +import com.alipay.oceanbase.rpc.location.model.partition.ObPair; +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; +import com.alipay.oceanbase.rpc.threadlocal.ThreadLocalMap; +import com.alipay.oceanbase.rpc.util.MonitorUtil; +import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; +import org.slf4j.Logger; +import com.alipay.oceanbase.rpc.table.api.TableQuery; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.LCD; +import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.RUNTIME; +import static com.alipay.oceanbase.rpc.protocol.payload.Constants.*; + +public class ObTableClientLSBatchOpsImpl extends AbstractTableBatchOps { + + private static final Logger logger = TableClientLoggerFactory + .getLogger(ObTableClientBatchOpsImpl.class); + private final ObTableClient obTableClient; + private ExecutorService executorService; + private boolean returningAffectedEntity = false; + private boolean needAllProp = false; + private List batchOperation; + + /* + * Ob table client batch ops impl. + */ + public ObTableClientLSBatchOpsImpl(String tableName, ObTableClient obTableClient) { + this.tableName = tableName; + this.obTableClient = obTableClient; + this.executorService = obTableClient.getRuntimeBatchExecutor(); + this.batchOperation = new ArrayList<>(); + } + + /* + * Get ob table batch operation. + */ + @Override + public ObTableBatchOperation getObTableBatchOperation() { + return null; + } + + public List getSingleOperations() { + return batchOperation; + } + + /* + * Get. + */ + @Override + public void get(Object[] rowkeys, String[] columns) { + throw new FeatureNotSupportedException(); + } + + /* + * Update. + */ + @Override + public void update(Object[] rowkeys, String[] columns, Object[] values) { + throw new FeatureNotSupportedException(); + } + + /* + * Delete. + */ + @Override + public void delete(Object[] rowkeys) { + throw new FeatureNotSupportedException(); + } + + /* + * Insert. + */ + @Override + public void insert(Object[] rowkeys, String[] columns, Object[] values) { + throw new FeatureNotSupportedException(); + } + + /* + * Replace. + */ + @Override + public void replace(Object[] rowkeys, String[] columns, Object[] values) { + throw new FeatureNotSupportedException(); + } + + /* + * Insert or update. + */ + @Override + public void insertOrUpdate(Object[] rowkeys, String[] columns, Object[] values) { + throw new FeatureNotSupportedException(); + } + + /* + * Increment. + */ + @Override + public void increment(Object[] rowkeys, String[] columns, Object[] values, boolean withResult) { + throw new FeatureNotSupportedException(); + } + + /* + * Append. + */ + @Override + public void append(Object[] rowkeys, String[] columns, Object[] values, boolean withResult) { + throw new FeatureNotSupportedException(); + } + + /* + * Put. + */ + @Override + public void put(Object[] rowkeys, String[] columns, Object[] values) { + throw new FeatureNotSupportedException(); + } + + private void addOperation(ObTableSingleOp singleOp) { + batchOperation.add(singleOp); + } + + public void addOperation(CheckAndInsUp checkAndInsUp) { + InsertOrUpdate insUp = checkAndInsUp.getInsUp(); + + ObTableSingleOpQuery query = new ObTableSingleOpQuery(); + ObNewRange range = new ObNewRange(); + range.setStartKey(ObRowKey.getInstance(insUp.getRowKeyValues())); + range.setEndKey(ObRowKey.getInstance(insUp.getRowKeyValues())); + query.addScanRangeColumns(insUp.getRowKeyNames()); + query.addScanRange(range); + query.setFilterString(checkAndInsUp.getFilter().toString()); + + String[] rowKeyNames = checkAndInsUp.getInsUp().getRowKeyNames().toArray(new String[0]); + Object[] rowKeyValues = checkAndInsUp.getInsUp().getRowKeyValues().toArray(new Object[0]); + String[] propertiesNames = checkAndInsUp.getInsUp().getColumns(); + Object[] propertiesValues = checkAndInsUp.getInsUp().getValues(); + ObTableSingleOpEntity entity = ObTableSingleOpEntity.getInstance(rowKeyNames, rowKeyValues, + propertiesNames, propertiesValues); + + ObTableSingleOp singleOp = new ObTableSingleOp(); + singleOp.setSingleOpType(ObTableOperationType.CHECK_AND_INSERT_UP); + singleOp.setIsCheckNoExists(!checkAndInsUp.isCheckExists()); + singleOp.setQuery(query); + singleOp.addEntity(entity); + + addOperation(singleOp); + } + + public void addOperation(TableQuery query) throws Exception { + // entity + String[] rowKeyNames = query.getRowKey().getColumns(); + if (rowKeyNames == null || rowKeyNames.length == 0) { + throw new IllegalArgumentException("rowKey is empty in get op"); + } + Object[] rowKey = query.getRowKey().getValues(); + String[] propertiesNames = query.getSelectColumns().toArray(new String[0]); + ObTableSingleOpEntity entity = ObTableSingleOpEntity.getInstance(rowKeyNames, rowKey, + propertiesNames, null); + if (propertiesNames.length == 0) { + needAllProp = true; + } + ObTableSingleOp singleOp = new ObTableSingleOp(); + singleOp.setSingleOpType(ObTableOperationType.GET); + singleOp.addEntity(entity); + addOperation(singleOp); + } + + public void addOperation(Mutation mutation) throws Exception { + // entity + String[] rowKeyNames = null; + Object[] rowKeyValues = null; + String[] propertiesNames = null; + Object[] propertiesValues = null; + + ObTableOperationType type = mutation.getOperationType(); + switch (type) { + case GET: + throw new IllegalArgumentException("Invalid type in batch operation, " + type); + case INSERT: + ((Insert) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Insert) mutation).getRowKeyNames().toArray(new String[0]); + rowKeyValues = ((Insert) mutation).getRowKeyValues().toArray(new Object[0]); + propertiesNames = ((Insert) mutation).getColumns(); + propertiesValues = ((Insert) mutation).getValues(); + break; + case DEL: + rowKeyNames = ((Delete) mutation).getRowKeyNames().toArray(new String[0]); + rowKeyValues = ((Delete) mutation).getRowKeyValues().toArray(new Object[0]); + break; + case UPDATE: + ((Update) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Update) mutation).getRowKeyNames().toArray(new String[0]); + rowKeyValues = ((Update) mutation).getRowKeyValues().toArray(new Object[0]); + propertiesNames = ((Update) mutation).getColumns(); + propertiesValues = ((Update) mutation).getValues(); + break; + case INSERT_OR_UPDATE: + ((InsertOrUpdate) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((InsertOrUpdate) mutation).getRowKeyNames().toArray(new String[0]); + rowKeyValues = ((InsertOrUpdate) mutation).getRowKeyValues().toArray(new Object[0]); + propertiesNames = ((InsertOrUpdate) mutation).getColumns(); + propertiesValues = ((InsertOrUpdate) mutation).getValues(); + break; + case REPLACE: + ((Replace) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Replace) mutation).getRowKeyNames().toArray(new String[0]); + rowKeyValues = ((Replace) mutation).getRowKeyValues().toArray(new Object[0]); + propertiesNames = ((Replace) mutation).getColumns(); + propertiesValues = ((Replace) mutation).getValues(); + break; + case INCREMENT: + ((Increment) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Increment) mutation).getRowKeyNames().toArray(new String[0]); + rowKeyValues = ((Increment) mutation).getRowKeyValues().toArray(new Object[0]); + propertiesNames = ((Increment) mutation).getColumns(); + propertiesValues = ((Increment) mutation).getValues(); + break; + case APPEND: + ((Append) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Append) mutation).getRowKeyNames().toArray(new String[0]); + rowKeyValues = ((Append) mutation).getRowKeyValues().toArray(new Object[0]); + propertiesNames = ((Append) mutation).getColumns(); + propertiesValues = ((Append) mutation).getValues(); + break; + case PUT: + ((Put) mutation).removeRowkeyFromMutateColval(); + rowKeyNames = ((Put) mutation).getRowKeyNames().toArray(new String[0]); + rowKeyValues = ((Put) mutation).getRowKeyValues().toArray(new Object[0]); + propertiesNames = ((Put) mutation).getColumns(); + propertiesValues = ((Put) mutation).getValues(); + break; + default: + throw new ObTableException("unknown operation type " + type); + } + + ObTableSingleOpEntity entity = ObTableSingleOpEntity.getInstance(rowKeyNames, rowKeyValues, + propertiesNames, propertiesValues); + ObTableSingleOp singleOp = new ObTableSingleOp(); + singleOp.setSingleOpType(type); + singleOp.addEntity(entity); + addOperation(singleOp); + } + + /* + * Execute. + */ + public List execute() throws Exception { + List results = new ArrayList(batchOperation.size()); + for (ObTableSingleOpResult result : executeInternal()) { + int errCode = result.getHeader().getErrno(); + if (errCode == ResultCodes.OB_SUCCESS.errorCode) { + results.add(result.getAffectedRows()); + } else { + results.add(ExceptionUtil.convertToObTableException(result.getExecuteHost(), + result.getExecutePort(), result.getSequence(), result.getUniqueId(), errCode, + result.getHeader().getErrMsg())); + } + } + return results; + } + + /* + * Execute with result + */ + public List executeWithResult() throws Exception { + List results = new ArrayList(batchOperation.size()); + for (ObTableSingleOpResult result : executeInternal()) { + int errCode = result.getHeader().getErrno(); + if (errCode == ResultCodes.OB_SUCCESS.errorCode) { + results.add(new MutationResult(result)); + } else { + results.add(ExceptionUtil.convertToObTableException(result.getExecuteHost(), + result.getExecutePort(), result.getSequence(), result.getUniqueId(), errCode, + result.getHeader().getErrMsg())); + } + } + return results; + } + + public Map>>>> partitionPrepare() + throws Exception { + List operations = getSingleOperations(); + // map: >>> + Map>>>> lsOperationsMap = + new HashMap(); + + // In ODP mode, client send the request to ODP directly without route + if (obTableClient.isOdpMode()) { + Map>>> tabletOperationsMap = new HashMap<>(); + ObPair>> obTableOperations = + new ObPair(new ObTableParam(obTableClient.getOdpTable()), + new ArrayList>()); + for (int i = 0; i < operations.size(); i++) { + ObTableSingleOp operation = operations.get(i); + obTableOperations.getRight().add(new ObPair(i, operation)); + } + tabletOperationsMap.put(INVALID_TABLET_ID, obTableOperations); + lsOperationsMap.put(INVALID_LS_ID, tabletOperationsMap); + return lsOperationsMap; + } + + for (int i = 0; i < operations.size(); i++) { + ObTableSingleOp operation = operations.get(i); + List rowkeyObjs = operation.getRowkeyObjs(); + int rowKeySize = rowkeyObjs.size(); + Object[] rowKey = new Object[rowKeySize]; + for (int j = 0; j < rowKeySize; j++) { + rowKey[j] = rowkeyObjs.get(j).getValue(); + } + ObPair tableObPair= obTableClient.getTable(tableName, rowKey, + false, false, obTableClient.getRoute(false)); + long lsId = tableObPair.getRight().getLsId(); + + Map>>> tabletOperations + = lsOperationsMap.get(lsId); + // if ls id not exists + if (tabletOperations == null) { + tabletOperations = new HashMap<>(); + lsOperationsMap.put(lsId, tabletOperations); + } + + ObPair>> singleOperations = + tabletOperations.get(tableObPair.getLeft()); + // if tablet id not exists + if (singleOperations == null) { + singleOperations = new ObPair<>(tableObPair.getRight(), new ArrayList<>()); + tabletOperations.put(tableObPair.getLeft(), singleOperations); + } + + singleOperations.getRight().add(new ObPair(i, operation)); + } + + return lsOperationsMap; + } + + /* + * Partition execute. + */ + public void partitionExecute(ObTableSingleOpResult[] results, + Map.Entry>>>> lsOperation) + throws Exception { + long lsId = lsOperation.getKey(); + Map>>> tabletOperationsMap = lsOperation.getValue(); + if (tabletOperationsMap.size() == 0) { + logger.warn("the size of tablet operations in ls operation is zero"); + throw new ObTableUnexpectedException("the size of tablet operations in ls operation is zero"); + } + + ObTableLSOpRequest tableLsOpRequest = new ObTableLSOpRequest(); + ObTableLSOperation tableLsOp = new ObTableLSOperation(); + tableLsOp.setLsId(lsId); + tableLsOp.setReturnOneResult(returnOneResult); + tableLsOp.setNeedAllProp(needAllProp); + tableLsOp.setTableName(tableName); + // fetch the following parameters in first entry for routing + long tableId = 0; + long originPartId = 0; + long operationTimeout = 0; + ObTable subObTable = null; + + boolean isFirstEntry = true; + // list ( index list for tablet op 1, index list for tablet op 2, ...) + List>> lsOperationWithIndexList = new ArrayList<>(); + for (final Map.Entry>>> tabletOperation : tabletOperationsMap.entrySet()) { + ObTableParam tableParam = tabletOperation.getValue().getLeft(); + long tabletId = tableParam.getPartitionId(); + List> tabletOperationWithIndexList = tabletOperation.getValue().getRight(); + lsOperationWithIndexList.add(tabletOperationWithIndexList); + List singleOps = new ArrayList<>(); + for (ObPair operationWithIndex : tabletOperationWithIndexList) { + singleOps.add(operationWithIndex.getRight()); + } + ObTableTabletOp tableTabletOp = new ObTableTabletOp(); + tableTabletOp.setSingleOperations(singleOps); + tableTabletOp.setTabletId(tabletId); + + tableLsOp.addTabletOperation(tableTabletOp); + + if (isFirstEntry) { + tableId = tableParam.getTableId(); + originPartId = tableParam.getPartId(); + operationTimeout = tableParam.getObTable().getObTableOperationTimeout(); + subObTable = tableParam.getObTable(); + isFirstEntry = false; + } + } + + // Since we only have one tablet operation + // We do the LS operation prepare here + tableLsOp.prepare(); + + tableLsOpRequest.setLsOperation(tableLsOp); + tableLsOpRequest.setTableId(tableId); + tableLsOpRequest.setEntityType(entityType); + tableLsOpRequest.setTimeout(operationTimeout); + + ObTableLSOpResult subLSOpResult; + boolean needRefreshTableEntry = false; + int tryTimes = 0; + long startExecute = System.currentTimeMillis(); + Set failedServerList = null; + ObServerRoute route = null; + + while (true) { + obTableClient.checkStatus(); + long currentExecute = System.currentTimeMillis(); + long costMillis = currentExecute - startExecute; + if (costMillis > obTableClient.getRuntimeMaxWait()) { + logger.error("table name: {} ls id:{} it has tried " + tryTimes + + " times and it has waited " + costMillis + " ms" + + " which exceeds runtime max wait timeout " + + obTableClient.getRuntimeMaxWait() + " ms", tableName, lsId); + throw new ObTableTimeoutExcetion("it has tried " + tryTimes + + " times and it has waited " + costMillis + + "ms which exceeds runtime max wait timeout " + + obTableClient.getRuntimeMaxWait() + "ms"); + } + tryTimes++; + try { + if (obTableClient.isOdpMode()) { + subObTable = obTableClient.getOdpTable(); + } else { + if (tryTimes > 1) { + if (route == null) { + route = obTableClient.getRoute(false); + } + if (failedServerList != null) { + route.setBlackList(failedServerList); + } + subObTable = obTableClient.getTableWithPartId(tableName, originPartId, needRefreshTableEntry, + obTableClient.isTableEntryRefreshIntervalWait(), false, route). + getRight().getObTable(); + } + } + subLSOpResult = (ObTableLSOpResult) subObTable.execute(tableLsOpRequest); + obTableClient.resetExecuteContinuousFailureCount(tableName); + break; + } catch (Exception ex) { + if (obTableClient.isOdpMode()) { + logger.warn("meet exception when execute ls batch in odp mode." + + "tablename: {}, errMsg: {}", tableName, ex.getMessage()); + throw ex; + } else if (ex instanceof ObTableReplicaNotReadableException) { + if ((tryTimes - 1) < obTableClient.getRuntimeRetryTimes()) { + logger.warn("tablename:{} ls id:{} retry when replica not readable: {}", + tableName, lsId, ex.getMessage()); + if (failedServerList == null) { + failedServerList = new HashSet(); + } + failedServerList.add(subObTable.getIp()); + } else { + logger.warn("exhaust retry when replica not readable: {}", ex.getMessage()); + throw ex; + } + } else if (ex instanceof ObTableException + && ((ObTableException) ex).isNeedRefreshTableEntry()) { + needRefreshTableEntry = true; + logger.warn("tablename:{} ls id:{} batch ops refresh table while meet ObTableMasterChangeException, errorCode: {}", + tableName, lsId, ((ObTableException) ex).getErrorCode(), ex); + if (obTableClient.isRetryOnChangeMasterTimes() + && (tryTimes - 1) < obTableClient.getRuntimeRetryTimes()) { + logger.warn("tablename:{} ls id:{} batch ops retry while meet ObTableMasterChangeException, errorCode: {} , retry times {}", + tableName, lsId, ((ObTableException) ex).getErrorCode(), + tryTimes, ex); + } else { + obTableClient.calculateContinuousFailure(tableName, ex.getMessage()); + throw ex; + } + } else { + obTableClient.calculateContinuousFailure(tableName, ex.getMessage()); + throw ex; + } + } + Thread.sleep(obTableClient.getRuntimeRetryInterval()); + } + + long endExecute = System.currentTimeMillis(); + + if (subLSOpResult == null) { + RUNTIME.error("tablename:{} ls id:{} check batch operation result error: client get unexpected NULL result", + tableName, lsId); + throw new ObTableUnexpectedException("check batch operation result error: client get unexpected NULL result"); + } + + List tabletOpResults = subLSOpResult.getResults(); + int affectedRows = 0; + + if (returnOneResult) { + if (results[0] == null) { + results[0] = new ObTableSingleOpResult(); + } + + ObTableSingleOpResult singleOpResult = tabletOpResults.get(0).getResults().get(0); + if (singleOpResult.getHeader().getErrno() != ResultCodes.OB_SUCCESS.errorCode) { + results[0].getHeader().setErrno(singleOpResult.getHeader().getErrno()); + results[0].getHeader().setMsg(singleOpResult.getHeader().getMsg()); + } + results[0].setAffectedRows(results[0].getAffectedRows() + + tabletOpResults.get(0).getResults().get(0).getAffectedRows()); + } else { + for (int i = 0; i < tabletOpResults.size(); i++) { + List singleOpResults = tabletOpResults.get(i).getResults(); + for (int j = 0; j < singleOpResults.size(); j++) { + affectedRows += singleOpResults.size(); + } + List> singleOperationsWithIndexList = lsOperationWithIndexList.get(i); + if (singleOpResults.size() < singleOperationsWithIndexList.size()) { + // only one result when it across failed + // only one result when hkv puts + if (singleOpResults.size() == 1 && entityType == ObTableEntityType.HKV) { + ObTableSingleOpResult subObTableSingleOpResult = singleOpResults.get(0); + subObTableSingleOpResult.setExecuteHost(subObTable.getIp()); + subObTableSingleOpResult.setExecutePort(subObTable.getPort()); + for (ObPair SubOperationWithIndexList : singleOperationsWithIndexList) { + results[SubOperationWithIndexList.getLeft()] = subObTableSingleOpResult; + } + } else { + throw new IllegalArgumentException( + "check batch operation result size error: operation size [" + + singleOperationsWithIndexList.size() + "] result size [" + + singleOpResults.size() + "]"); + } + } else { + if (singleOpResults.size() != singleOperationsWithIndexList.size()) { + throw new ObTableUnexpectedException("check batch result error: ls " + + lsId + " expect result size " + + singleOperationsWithIndexList.size() + + " actual result size " + + singleOpResults.size() + + " for " + i + "th tablet operation"); + } + for (int j = 0; j < singleOperationsWithIndexList.size(); j++) { + ObTableSingleOpResult subObTableSingleOpResult = singleOpResults.get(j); + subObTableSingleOpResult.setExecuteHost(subObTable.getIp()); + subObTableSingleOpResult.setExecutePort(subObTable.getPort()); + results[singleOperationsWithIndexList.get(j).getLeft()] = subObTableSingleOpResult; + } + } + } + } + + + String endpoint = subObTable.getIp() + ":" + subObTable.getPort(); + MonitorUtil.info(tableLsOpRequest, subObTable.getDatabase(), tableName, + "LS_BATCH-Execute-", endpoint, tableLsOp, + affectedRows, endExecute - startExecute, + obTableClient.getslowQueryMonitorThreshold()); + } + + /* + * Execute internal. + */ + public ObTableSingleOpResult[] executeInternal() throws Exception { + + if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + long start = System.currentTimeMillis(); + ObTableSingleOpResult[] obTableOperationResults = null; + if (returnOneResult) { + obTableOperationResults = new ObTableSingleOpResult[1]; + } else { + obTableOperationResults = new ObTableSingleOpResult[batchOperation.size()]; + } + Map>>>> lsOperations = partitionPrepare(); + long getTableTime = System.currentTimeMillis(); + final Map context = ThreadLocalMap.getContextMap(); + if (executorService != null && !executorService.isShutdown() && lsOperations.size() > 1) { + // execute sub-batch operation in parallel + final ConcurrentTaskExecutor executor = new ConcurrentTaskExecutor(executorService, + lsOperations.size()); + for (final Map.Entry>>>> entry : lsOperations + .entrySet()) { + ObTableSingleOpResult[] finalObTableOperationResults = obTableOperationResults; + executor.execute(new ConcurrentTask() { + /* + * Do task. + */ + @Override + public void doTask() { + try { + ThreadLocalMap.transmitContextMap(context); + partitionExecute(finalObTableOperationResults, entry); + } catch (Exception e) { + logger.error(LCD.convert("01-00026"), e); + executor.collectExceptions(e); + } finally { + ThreadLocalMap.reset(); + } + } + }); + } + long timeoutTs = obTableClient.getRuntimeBatchMaxWait() * 1000L * 1000L + + System.nanoTime(); + try { + while (timeoutTs > System.nanoTime()) { + try { + executor.waitComplete(1, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new ObTableUnexpectedException( + "Batch Concurrent Execute interrupted", e); + } + + if (executor.getThrowableList().size() > 0) { + throw new ObTableUnexpectedException("Batch Concurrent Execute Error", + executor.getThrowableList().get(0)); + } + + if (executor.isComplete()) { + break; + } + } + } finally { + executor.stop(); + } + + if (executor.getThrowableList().size() > 0) { + throw new ObTableUnexpectedException("Batch Concurrent Execute Error", executor + .getThrowableList().get(0)); + } + + if (!executor.isComplete()) { + throw new ObTableUnexpectedException( + "Batch Concurrent Execute Error, runtimeBatchMaxWait: " + + obTableClient.getRuntimeBatchMaxWait() + "ms"); + } + + } else { + // Execute sub-batch operation one by one + for (final Map.Entry>>>> entry : lsOperations + .entrySet()) { + partitionExecute(obTableOperationResults, entry); + } + } + + if (obTableOperationResults.length <= 0) { + throw new ObTableUnexpectedException( + "Ls batch execute returns zero single operation results"); + } + + MonitorUtil + .info(obTableOperationResults[0], obTableClient.getDatabase(), tableName, "LS_BATCH", + "", obTableOperationResults.length, getTableTime - start, + System.currentTimeMillis() - getTableTime, + obTableClient.getslowQueryMonitorThreshold()); + + return obTableOperationResults; + } + + /* + * clear batch operations1 + */ + public void clear() { + batchOperation = new ArrayList<>(); + } + + /* + * Set executor service. + */ + public void setExecutorService(ExecutorService executorService) { + this.executorService = executorService; + } + + public boolean isReturningAffectedEntity() { + return returningAffectedEntity; + } + + public void setReturningAffectedEntity(boolean returningAffectedEntity) { + this.returningAffectedEntity = returningAffectedEntity; + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java index 87573b78..ecf435d0 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientQueryImpl.java @@ -18,41 +18,73 @@ package com.alipay.oceanbase.rpc.table; import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.location.model.partition.ObPair; +import com.alipay.oceanbase.rpc.mutation.Row; +import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation.ObTableAggregationType; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; +import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.stream.QueryResultSet; import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.MonitorUtil; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; public class ObTableClientQueryImpl extends AbstractTableQueryImpl { - private final String tableName; - private final ObTableClient obTableClient; + private String tableName; + private final ObTableClient obTableClient; + private Map> partitionObTables; - /** + private Row rowKey; // only used by BatchOperation + + /* + * Add aggregation. + */ + public void addAggregation(ObTableAggregationType aggType, String aggColumn) { + this.tableQuery.addAggregation(aggType, aggColumn); + } + + /* + * Ob table client query impl construct only with tableName + */ + public ObTableClientQueryImpl() { + this.tableName = null; + this.indexTableName = null; + this.obTableClient = null; + this.tableQuery = new ObTableQuery(); + this.rowKey = null; + } + + /* * Ob table client query impl. */ public ObTableClientQueryImpl(String tableName, ObTableClient client) { this.tableName = tableName; + this.indexTableName = tableName; this.obTableClient = client; this.tableQuery = new ObTableQuery(); + this.rowKey = null; } - /** + /* * Ob table client query impl. */ public ObTableClientQueryImpl(String tableName, ObTableQuery tableQuery, ObTableClient client) { this.tableName = tableName; + this.indexTableName = tableName; this.obTableClient = client; this.tableQuery = tableQuery; + this.rowKey = null; } - /** + /* * Execute. */ @Override @@ -60,22 +92,165 @@ public QueryResultSet execute() throws Exception { return new QueryResultSet(executeInternal()); } + /* + * Execute. + */ + @Override + public QueryResultSet asyncExecute() throws Exception { + return new QueryResultSet(asyncExecuteInternal()); + } + /** * 只有 limit query 需要，其他不需要 - * @param keys - * @return + * @param keys keys want to set + * @return table query */ @Override public TableQuery setKeys(String... keys) { throw new IllegalArgumentException("Not needed"); } - /** + /* + * set row key of query (only used by BatchOperation) + */ + public TableQuery setRowKey(Row rowKey) throws Exception { + this.rowKey = rowKey; + return this; + } + + /* + * check argument before execution + */ + public void checkArgumentBeforeExec() throws Exception { + if (null == obTableClient) { + throw new ObTableException("table client is null"); + } else if (tableQuery.getLimit() < 0 && tableQuery.getOffset() > 0) { + throw new ObTableException("offset can not be use without limit"); + } else if (tableName == null || tableName.isEmpty()) { + throw new IllegalArgumentException("table name is null"); + } + } + + /* + * Set parameter into request + */ + private void setCommonParams2Result(AbstractQueryStreamResult result) throws Exception { + result.setTableQuery(tableQuery); + result.setEntityType(entityType); + result.setTableName(tableName); + result.setIndexTableName(indexTableName); + result.setExpectant(partitionObTables); + result.setOperationTimeout(operationTimeout); + result.setReadConsistency(obTableClient.getReadConsistency()); + } + + private abstract static class InitQueryResultCallback { + abstract T execute() throws Exception; + } + + private AbstractQueryStreamResult commonExecute(InitQueryResultCallback callable) + throws Exception { + checkArgumentBeforeExec(); + + final long startTime = System.currentTimeMillis(); + this.partitionObTables = new LinkedHashMap>(); // partitionObTables -> Map> + + // fill a whole range if no range is added explicitly. + if (tableQuery.getKeyRanges().isEmpty()) { + tableQuery.addKeyRange(ObNewRange.getWholeRange()); + } + + // init partitionObTables + if (obTableClient.isOdpMode()) { + if (tableQuery.getScanRangeColumns().isEmpty()) { + if (tableQuery.getIndexName() != null + && !tableQuery.getIndexName().equalsIgnoreCase("primary")) { + throw new ObTableException("key range columns must be specified when use index"); + } + } + this.partitionObTables.put(0L, new ObPair(0L, new ObTableParam( + obTableClient.getOdpTable()))); + } else { + initPartitions(); + } + + StringBuilder stringBuilder = new StringBuilder(); + for (Map.Entry> entry : this.partitionObTables.entrySet()) { + stringBuilder.append("#").append(entry.getValue().getRight().getObTable().getIp()) + .append(":").append(entry.getValue().getRight().getObTable().getPort()); + } + String endpoint = stringBuilder.toString(); + long getTableTime = System.currentTimeMillis(); + + // defend aggregation of multiple partitions. + if (tableQuery.isAggregation()) { + if (this.partitionObTables.size() > 1) { + throw new ObTableException( + "Not supported aggregate of multiple partitions, the partition size is: " + + this.partitionObTables.size(), ResultCodes.OB_NOT_SUPPORTED.errorCode); + } + } + + // set correct table group name for hbase + if (tableQuery.isHbaseQuery() + && obTableClient.getTableGroupInverted().containsKey(tableName) + && tableName.equalsIgnoreCase(obTableClient.getTableGroupCache().get( + obTableClient.getTableGroupInverted().get(tableName)))) { + tableName = obTableClient.getTableGroupInverted().get(tableName); + } + + // init query stream result + AbstractQueryStreamResult streamResult = callable.execute(); + + MonitorUtil + .info((ObPayload) streamResult, obTableClient.getDatabase(), tableName, "QUERY", + endpoint, tableQuery, streamResult, getTableTime - startTime, + System.currentTimeMillis() - getTableTime, + obTableClient.getslowQueryMonitorThreshold()); + + return streamResult; + } + + /* * Execute internal. */ public ObTableClientQueryStreamResult executeInternal() throws Exception { - Map> partitionObTables = new HashMap>(); - for (ObNewRange rang : tableQuery.getKeyRanges()) { + return (ObTableClientQueryStreamResult) commonExecute(new InitQueryResultCallback() { + @Override + ObTableClientQueryStreamResult execute() throws Exception { + ObTableClientQueryStreamResult obTableClientQueryStreamResult = new ObTableClientQueryStreamResult(); + setCommonParams2Result(obTableClientQueryStreamResult); + obTableClientQueryStreamResult.setClient(obTableClient); + obTableClientQueryStreamResult.init(); + return obTableClientQueryStreamResult; + } + }); + } + + public ObTableClientQueryAsyncStreamResult asyncExecuteInternal() throws Exception { + return (ObTableClientQueryAsyncStreamResult) commonExecute(new InitQueryResultCallback() { + @Override + ObTableClientQueryAsyncStreamResult execute() throws Exception { + ObTableClientQueryAsyncStreamResult obTableClientQueryAsyncStreamResult = new ObTableClientQueryAsyncStreamResult(); + setCommonParams2Result(obTableClientQueryAsyncStreamResult); + obTableClientQueryAsyncStreamResult.setClient(obTableClient); + obTableClientQueryAsyncStreamResult.init(); + return obTableClientQueryAsyncStreamResult; + } + }); + } + + /* + * Init partition tables involved in this query + */ + public void initPartitions() throws Exception { + String indexName = tableQuery.getIndexName(); + if (!this.obTableClient.isOdpMode()) { + indexTableName = obTableClient.getIndexTableName(tableName, indexName, + tableQuery.getScanRangeColumns(), false); + } + + for (ObNewRange rang : this.tableQuery.getKeyRanges()) { ObRowKey startKey = rang.getStartKey(); int startKeySize = startKey.getObjs().size(); ObRowKey endKey = rang.getEndKey(); @@ -83,35 +258,37 @@ public ObTableClientQueryStreamResult executeInternal() throws Exception { Object[] start = new Object[startKeySize]; Object[] end = new Object[endKeySize]; for (int i = 0; i < startKeySize; i++) { - start[i] = startKey.getObj(i).getValue(); + if (startKey.getObj(i).isMinObj() || startKey.getObj(i).isMaxObj()) { + start[i] = startKey.getObj(i); + } else { + start[i] = startKey.getObj(i).getValue(); + } } - for (int i = 0; i < endKeySize; i++) { - end[i] = endKey.getObj(i).getValue(); + if (endKey.getObj(i).isMinObj() || endKey.getObj(i).isMaxObj()) { + end[i] = endKey.getObj(i); + } else { + end[i] = endKey.getObj(i).getValue(); + } } ObBorderFlag borderFlag = rang.getBorderFlag(); - List> pairs = obTableClient.getTables(tableName, start, - borderFlag.isInclusiveStart(), end, borderFlag.isInclusiveEnd(), false, false, - obTableClient.getReadRoute()); - for (ObPair pair : pairs) { - partitionObTables.put(pair.getLeft(), pair); + // pairs -> List> + List> pairs = this.obTableClient.getTables(indexTableName, + tableQuery, start, borderFlag.isInclusiveStart(), end, borderFlag.isInclusiveEnd(), + false, false); + if (this.tableQuery.getScanOrder() == ObScanOrder.Reverse) { + for (int i = pairs.size() - 1; i >= 0; i--) { + this.partitionObTables.put(pairs.get(i).getLeft(), pairs.get(i)); + } + } else { + for (ObPair pair : pairs) { + this.partitionObTables.put(pair.getLeft(), pair); + } } } - - ObTableClientQueryStreamResult obTableClientQueryStreamResult = new ObTableClientQueryStreamResult(); - obTableClientQueryStreamResult.setTableQuery(tableQuery); - obTableClientQueryStreamResult.setEntityType(entityType); - obTableClientQueryStreamResult.setTableName(tableName); - obTableClientQueryStreamResult.setExpectant(partitionObTables); - obTableClientQueryStreamResult.setClient(obTableClient); - obTableClientQueryStreamResult.setOperationTimeout(operationTimeout); - obTableClientQueryStreamResult.setReadConsistency(obTableClient.getReadConsistency()); - obTableClientQueryStreamResult.init(); - - return obTableClientQueryStreamResult; } - /** + /* * Clear. */ @Override @@ -119,7 +296,7 @@ public void clear() { this.tableQuery = new ObTableQuery(); } - /** + /* * Get ob table query. */ @Override @@ -127,10 +304,21 @@ public ObTableQuery getObTableQuery() { return tableQuery; } - /** + /* * Get table name. */ public String getTableName() { return tableName; } + + public void setTableName(String tabName) { + this.tableName = tabName; + } + + /* + * Get row key (used by BatchOperation) + */ + public Row getRowKey() { + return this.rowKey; + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableDirectLoad.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableDirectLoad.java new file mode 100644 index 00000000..f0460599 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableDirectLoad.java @@ -0,0 +1,368 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.table; + +import com.alipay.oceanbase.rpc.ObGlobal; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.property.AbstractPropertyAware; +import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; +import com.alipay.oceanbase.rpc.protocol.payload.ObSimplePayload; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObAddr; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadAbortArg; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadBeginArg; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadBeginRes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadCommitArg; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadGetStatusArg; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadGetStatusRes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadInsertArg; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadHeartBeatArg; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadHeartBeatRes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadRequest; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableDirectLoadResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.direct_load.ObTableLoadClientStatus; +import com.alipay.oceanbase.rpc.util.ObBytesString; +import com.alipay.oceanbase.rpc.util.TableClientLoggerFactory; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; + +import java.util.Properties; +import java.util.Timer; +import java.util.TimerTask; + +import org.slf4j.Logger; + +import static com.alipay.oceanbase.rpc.property.Property.*; + +public class ObTableDirectLoad extends AbstractPropertyAware { + + private static final Logger logger = TableClientLoggerFactory + .getLogger(ObTableDirectLoad.class); + + private ObTable table = null; + private String tableName; + private ObDirectLoadParameter parameter = null; + private boolean forceCreate = false; + + private long tableId = 0; + private long taskId = 0; + private String[] columnNames = new String[0]; + private ObTableLoadClientStatus status = ObTableLoadClientStatus.MAX_STATUS; + private int errorCode = ResultCodes.OB_SUCCESS.errorCode; + private ObAddr srvAddr = null; + + private Timer timer; + + /* Properities */ + protected int runtimeRetryTimes = RUNTIME_RETRY_TIMES.getDefaultInt(); + protected int runtimeRetryInterval = RUNTIME_RETRY_INTERVAL.getDefaultInt(); + + public ObTableDirectLoad(ObTable table, String tableName, ObDirectLoadParameter parameter, + boolean forceCreate) { + if (ObGlobal.OB_VERSION < ObGlobal.OB_VERSION_4_2_1_0) { + logger.warn("not supported ob version {}", ObGlobal.obVsnString()); + throw new ObTableException("not supported ob version " + ObGlobal.obVsnString(), + ResultCodes.OB_NOT_SUPPORTED.errorCode); + } + if (table == null || tableName == null || tableName.isEmpty() || parameter == null) { + logger.warn("invalid args, table:{}, tableName:{}, parameter:{}", table, tableName, + parameter); + throw new IllegalArgumentException("invalid args: " + table + ", " + tableName + ", " + + parameter); + } + this.table = table; + this.tableName = tableName; + this.parameter = parameter; + this.forceCreate = forceCreate; + initProperties(); + } + + @Override + public void setProperties(Properties properties) { + this.properties = properties; + initProperties(); + } + + public ObTable getTable() { + return table; + } + + public String getTableName() { + return tableName; + } + + public ObDirectLoadParameter getParameter() { + return parameter; + } + + public long getTableId() { + return tableId; + } + + public long getTaskId() { + return taskId; + } + + public String[] getColumnNames() { + return columnNames; + } + + public boolean isAvailable() { + return status != ObTableLoadClientStatus.MAX_STATUS; + } + + public boolean isRunning() { + return status == ObTableLoadClientStatus.RUNNING; + } + + public boolean isCommitting() { + return status == ObTableLoadClientStatus.COMMITTING; + } + + public boolean isCommit() { + return status == ObTableLoadClientStatus.COMMIT; + } + + public boolean isError() { + return status == ObTableLoadClientStatus.ERROR; + } + + public boolean isAbort() { + return status == ObTableLoadClientStatus.ABORT; + } + + public int getErrorCode() { + return errorCode; + } + + public ObAddr getSrvAddr() { + return srvAddr; + } + + public String toString() { + return String.format("{tableName:%s, tableId:%d, taskId:%d}", tableName, tableId, taskId); + } + + private void initProperties() { + runtimeRetryTimes = parseToInt(RUNTIME_RETRY_TIMES.getKey(), runtimeRetryTimes); + runtimeRetryInterval = parseToInt(RUNTIME_RETRY_INTERVAL.getKey(), runtimeRetryInterval); + } + + public void begin() throws Exception { + if (status != ObTableLoadClientStatus.MAX_STATUS) { + logger.warn("unexpected status to begin, table:{}, status:{}", this, status); + throw new ObTableException("unexpected status to begin, status:" + status, + ResultCodes.OB_STATE_NOT_MATCH.errorCode); + } + ObTableDirectLoadBeginArg arg = new ObTableDirectLoadBeginArg(); + ObTableDirectLoadBeginRes res = new ObTableDirectLoadBeginRes(); + arg.setTableName(tableName); + arg.setParallel(parameter.getParallel()); + arg.setMaxErrorRowCount(parameter.getMaxErrorRowCount()); + arg.setDupAction(parameter.getDupAction()); + arg.setTimeout(parameter.getTimeout()); + arg.setHeartBeatTimeout(parameter.getHeartBeatTimeout()); + arg.setForceCreate(forceCreate); + execute(ObTableDirectLoadOperationType.BEGIN, arg, res); + tableId = res.getTableId(); + taskId = res.getTaskId(); + columnNames = res.getColumnNames(); + status = res.getStatus(); + errorCode = res.getErrorCode(); + logger.info("begin suceess, table:{}", this); + startHeartBeat(); + } + + public void commit() throws Exception { + if (!isRunning()) { + logger.warn("unexpected status to commit, table:{}, status:{}", this, status); + throw new ObTableException("unexpected status to commit, status:" + status, + ResultCodes.OB_STATE_NOT_MATCH.errorCode); + } + ObTableDirectLoadCommitArg arg = new ObTableDirectLoadCommitArg(); + arg.setTableId(tableId); + arg.setTaskId(taskId); + execute(ObTableDirectLoadOperationType.COMMIT, arg); + logger.info("commit suceess, table:{}", this); + } + + public void abort() throws Exception { + if (status == ObTableLoadClientStatus.MAX_STATUS) { + logger.warn("unexpected status to abort, table:{}, status:{}", this, status); + throw new ObTableException("unexpected status to abort, status:" + status, + ResultCodes.OB_STATE_NOT_MATCH.errorCode); + } + if (isAbort()) { + return; + } + ObTableDirectLoadAbortArg arg = new ObTableDirectLoadAbortArg(); + arg.setTableId(tableId); + arg.setTaskId(taskId); + execute(ObTableDirectLoadOperationType.ABORT, arg); + logger.info("abort suceess, table:{}", this); + stopHeartBeat(); + } + + public ObTableLoadClientStatus getStatus() throws Exception { + if (status == ObTableLoadClientStatus.MAX_STATUS) { + logger.warn("unexpected status to get status, table:{}, status:{}", this, status); + throw new ObTableException("unexpected status to get status, status:" + status, + ResultCodes.OB_STATE_NOT_MATCH.errorCode); + } + ObTableDirectLoadGetStatusArg arg = new ObTableDirectLoadGetStatusArg(); + ObTableDirectLoadGetStatusRes res = new ObTableDirectLoadGetStatusRes(); + arg.setTableId(tableId); + arg.setTaskId(taskId); + execute(ObTableDirectLoadOperationType.GET_STATUS, arg, res); + status = res.getStatus(); + errorCode = res.getErrorCode(); + return status; + } + + public void insert(ObDirectLoadBucket bucket) throws Exception { + if (!isRunning()) { + logger.warn("unexpected status to insert, table:{}, status:{}", this, status); + throw new ObTableException("unexpected status to insert, status:" + status, + ResultCodes.OB_STATE_NOT_MATCH.errorCode); + } + ObTableDirectLoadInsertArg arg = new ObTableDirectLoadInsertArg(); + arg.setTableId(tableId); + arg.setTaskId(taskId); + arg.setPayload(new ObBytesString(bucket.encode())); + execute(ObTableDirectLoadOperationType.INSERT, arg); + } + + private void heartBeat() throws Exception { + ObTableDirectLoadHeartBeatArg arg = new ObTableDirectLoadHeartBeatArg(); + ObTableDirectLoadHeartBeatRes res = new ObTableDirectLoadHeartBeatRes(); + arg.setTableId(tableId); + arg.setTaskId(taskId); + execute(ObTableDirectLoadOperationType.HEART_BEAT, arg, res); + status = res.getStatus(); + errorCode = res.getErrorCode(); + } + + private void startHeartBeat() { + timer = new Timer(); + TimerTask task = new TimerTask() { + public void run() { + try { + heartBeat(); + } catch (Exception e) { + stopHeartBeat(); + logger.info(String.format("heart beat failed, table:%s", this), e); + } + } + }; + timer.schedule(task, 0, 10000); + logger.info("start heart beat, table:{}", this); + } + + private void stopHeartBeat() { + if (timer != null) { + timer.cancel(); + timer = null; + } + logger.info("stop heart beat, table:{}", this); + } + + private void execute(ObTableDirectLoadOperationType operationType, ObSimplePayload arg, + ObSimplePayload res) throws Exception { + ObTableDirectLoadRequest request = new ObTableDirectLoadRequest(); + ObTableDirectLoadRequest.Header header = new ObTableDirectLoadRequest.Header(); + if (operationType != ObTableDirectLoadOperationType.BEGIN) { + header.setAddr(srvAddr); + } + header.setOperationType(operationType); + request.setHeader(header); + request.setArgContent(new ObBytesString(arg.encode())); + ObTableDirectLoadResult result = (ObTableDirectLoadResult) rpcCall(request); + if (result.getHeader().getOperationType() != operationType) { + logger.warn("unexpected result operation type, table:{}, reqOpType:{}, resOpType:{}", + this, operationType, result.getHeader().getOperationType()); + throw new ObTableException("unexpected result operation type:" + + result.getHeader().getOperationType() + + ", request operation type:" + operationType, + ResultCodes.OB_ERR_UNEXPECTED.errorCode); + } + if (result.getResContent().length() == 0) { + logger.warn("unexpected empty res content, table:{}, OpType:{}", this, operationType); + throw new ObTableException("unexpected empty res content", + ResultCodes.OB_ERR_UNEXPECTED.errorCode); + } + if (operationType == ObTableDirectLoadOperationType.BEGIN) { + srvAddr = result.getHeader().getAddr(); + } + ByteBuf buf = ByteBufAllocator.DEFAULT.buffer(result.getResContent().length()); + try { + buf.writeBytes(result.getResContent().bytes); + res.decode(buf); + } finally { + buf.release(); + } + } + + private void execute(ObTableDirectLoadOperationType operationType, ObSimplePayload arg) + throws Exception { + ObTableDirectLoadRequest request = new ObTableDirectLoadRequest(); + ObTableDirectLoadRequest.Header header = new ObTableDirectLoadRequest.Header(); + if (operationType != ObTableDirectLoadOperationType.BEGIN) { + header.setAddr(srvAddr); + } + header.setOperationType(operationType); + request.setHeader(header); + request.setArgContent(new ObBytesString(arg.encode())); + ObTableDirectLoadResult result = (ObTableDirectLoadResult) rpcCall(request); + if (result.getHeader().getOperationType() != operationType) { + logger.warn("unexpected result operation type, table:{}, reqOpType:{}, resOpType:{}", + this, operationType, result.getHeader().getOperationType()); + throw new ObTableException("unexpected result operation type:" + + result.getHeader().getOperationType() + + ", request operation type:" + operationType, + ResultCodes.OB_ERR_UNEXPECTED.errorCode); + } + if (result.getResContent().length() != 0) { + logger.warn("unexpected res content not empty, table:{}, OpType:{}", this, + operationType); + throw new ObTableException("unexpected res content not empty", + ResultCodes.OB_ERR_UNEXPECTED.errorCode); + } + } + + private ObPayload rpcCall(ObTableDirectLoadRequest request) throws Exception { + int tries = 0; + while (true) { + try { + request.setTimeout(table.getObTableExecuteTimeout()); + return table.execute(request); + } catch (Exception e) { + logger.warn(String.format("table execute failed, table:%s, tries:%d, request:%s", + this, tries, request), e); + if (tries < runtimeRetryTimes) { + Thread.sleep(runtimeRetryInterval); + } else { + throw e; + } + ++tries; + } + } + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableParam.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableParam.java new file mode 100644 index 00000000..216382c3 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableParam.java @@ -0,0 +1,133 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.table; + +import com.alipay.oceanbase.rpc.ObTableClient; + +import static com.alipay.oceanbase.rpc.protocol.payload.Constants.INVALID_TABLET_ID; +import static com.alipay.oceanbase.rpc.protocol.payload.Constants.OB_INVALID_ID; +import static com.alipay.oceanbase.rpc.protocol.payload.Constants.INVALID_LS_ID; + +public class ObTableParam { + private ObTable obTable; + private long tableId = OB_INVALID_ID; + private long partitionId = INVALID_TABLET_ID; // partition id in 3.x aka tablet id in 4.x + private long partId = INVALID_TABLET_ID; // logicId, partition id in 3.x, can be used when retry + private long lsId = INVALID_LS_ID; + + /* + * constructor + */ + public ObTableParam(ObTable obTable, long tableId, long partitionId) { + this.obTable = obTable; + this.tableId = tableId; + this.partitionId = partitionId; + } + + /* + * constructor + */ + public ObTableParam(ObTable obTable) { + this.obTable = obTable; + } + + /* + * Get ObTable. + */ + public ObTable getObTable() { + return obTable; + } + + /* + * Set ObTable. + */ + public void setObTable(ObTable obTable) { + this.obTable = obTable; + } + + /* + * Get tableId. + */ + public long getTableId() { + return tableId; + } + + /* + * Set tableId. + */ + public void setTableId(long tableId) { + this.tableId = tableId; + } + + /* + * Get tabletId. + */ + public long getTabletId() { + return partitionId; + } + + /* + * Set tabletId. partition ID is the same as tablet ID. + */ + public void setTabletId(long tabletId) { + this.partitionId = tabletId; + } + + /* + * Get partitionId. + */ + public long getPartitionId() { + return partitionId; + } + + /* + * Set partitionId. + */ + public void setPartitionId(long partitionId) { + this.partitionId = partitionId; + } + + /* + * Get partId (partition id in 3.x, originPartId in 4.x) + */ + public long getPartId() { + return this.partId; + } + + /* + * Set partId + */ + public void setPartId(long partId) { + this.partId = partId; + } + + /* + * Set lsId + */ + public long getLsId() { + return lsId; + } + + /* + * Get lsId + */ + public void setLsId(long lsId) { + this.lsId = lsId; + } + +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableQueryImpl.java b/src/main/java/com/alipay/oceanbase/rpc/table/ObTableQueryImpl.java deleted file mode 100644 index 606a6864..00000000 --- a/src/main/java/com/alipay/oceanbase/rpc/table/ObTableQueryImpl.java +++ /dev/null @@ -1,121 +0,0 @@ -/*- - * #%L - * OBKV Table Client Framework - * %% - * Copyright (C) 2021 OceanBase - * %% - * OBKV Table Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.rpc.table; - -import com.alipay.oceanbase.rpc.location.model.partition.ObPair; -import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; -import com.alipay.oceanbase.rpc.stream.ObTableQueryStreamResult; -import com.alipay.oceanbase.rpc.stream.QueryResultSet; -import com.alipay.oceanbase.rpc.table.api.TableQuery; - -import java.util.HashMap; -import java.util.Map; - -public class ObTableQueryImpl extends AbstractTableQueryImpl { - - private final String tableName; - private ObTable table; - - private ObTableQueryRequest request; - - /** - * Ob table query impl. - */ - public ObTableQueryImpl(String tableName, ObTable table) { - this.tableName = tableName; - this.table = table; - - resetRequest(); - } - - private void resetRequest() { - this.request = new ObTableQueryRequest(); - this.tableQuery = new ObTableQuery(); - this.request.setTableName(tableName); - this.request.setTableQuery(tableQuery); - // FIXME TableQuery 必须设置 PartitionId - this.request.setPartitionId(0); - } - - /** - * Execute. - */ - @Override - public QueryResultSet execute() throws Exception { - Map> partitionObTable = new HashMap>(); - partitionObTable.put(0L, new ObPair(0L, table)); - ObTableQueryStreamResult obTableQueryStreamResult = new ObTableQueryStreamResult(); - obTableQueryStreamResult.setTableQuery(tableQuery); - obTableQueryStreamResult.setEntityType(entityType); - obTableQueryStreamResult.setTableName(tableName); - obTableQueryStreamResult.setExpectant(partitionObTable); - obTableQueryStreamResult.setOperationTimeout(operationTimeout); - obTableQueryStreamResult.init(); - return new QueryResultSet(obTableQueryStreamResult); - - } - - /** - * 只有 limit query 需要，其他不需要 - * @param keys - * @return - */ - @Override - public TableQuery setKeys(String... keys) { - throw new IllegalArgumentException("Not needed"); - } - - /** - * Clear. - */ - @Override - public void clear() { - resetRequest(); - } - - /** - * Get ob table query. - */ - @Override - public ObTableQuery getObTableQuery() { - return tableQuery; - } - - /** - * Get table name. - */ - @Override - public String getTableName() { - return tableName; - } - - /** - * Get table. - */ - public ObTable getTable() { - return table; - } - - /** - * Reset ob table. - */ - public void resetObTable(ObTable obTable) { - this.table = obTable; - } - -} diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/api/Table.java b/src/main/java/com/alipay/oceanbase/rpc/table/api/Table.java index bc2cec56..b8c30f1f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/api/Table.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/api/Table.java @@ -17,6 +17,10 @@ package com.alipay.oceanbase.rpc.table.api; +import com.alipay.oceanbase.rpc.checkandmutate.CheckAndInsUp; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; +import com.alipay.oceanbase.rpc.mutation.*; + import java.util.Map; public interface Table { @@ -25,54 +29,72 @@ public interface Table { TableQuery query(String tableName) throws Exception; - TableQuery queryByBatch(String tableName) throws Exception; - TableBatchOps batch(String tableName) throws Exception; Map get(String tableName, Object rowkey, String[] columns) throws Exception; Map get(String tableName, Object[] rowkeys, String[] columns) throws Exception; + Update update(String tableName); + long update(String tableName, Object rowkey, String[] columns, Object[] values) throws Exception; long update(String tableName, Object[] rowkeys, String[] columns, Object[] values) throws Exception; + Delete delete(String tableName); + long delete(String tableName, Object rowkey) throws Exception; long delete(String tableName, Object[] rowkeys) throws Exception; + public Insert insert(String tableName); + long insert(String tableName, Object rowkey, String[] columns, Object[] values) throws Exception; long insert(String tableName, Object[] rowkeys, String[] columns, Object[] values) throws Exception; + Replace replace(String tableName); + long replace(String tableName, Object rowkey, String[] columns, Object[] values) throws Exception; long replace(String tableName, Object[] rowkeys, String[] columns, Object[] values) throws Exception; + InsertOrUpdate insertOrUpdate(String tableName); + long insertOrUpdate(String tableName, Object rowkey, String[] columns, Object[] values) throws Exception; long insertOrUpdate(String tableName, Object[] rowkeys, String[] columns, Object[] values) throws Exception; + Put put(String tableName); + + Increment increment(String tableName); + Map increment(String tableName, Object rowkey, String[] columns, Object[] values, boolean withResult) throws Exception; Map increment(String tableName, Object[] rowkeys, String[] columns, Object[] values, boolean withResult) throws Exception; + Append append(String tableName); + Map append(String tableName, Object rowkey, String[] columns, Object[] values, boolean withResult) throws Exception; Map append(String tableName, Object[] rowkeys, String[] columns, Object[] values, boolean withResult) throws Exception; + BatchOperation batchOperation(String tableName); + void addProperty(String property, String value); + CheckAndInsUp checkAndInsUp(String tableName, ObTableFilter filter, InsertOrUpdate insUp, + boolean checkExists); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java b/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java index dc9772a4..631aa8f9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/api/TableBatchOps.java @@ -32,6 +32,10 @@ public interface TableBatchOps { boolean isAtomicOperation(); + void setReturnOneResult(boolean returnOneResult); + + boolean isReturnOneResult(); + void setEntityType(ObTableEntityType entityType); ObTableEntityType getEntityType(); @@ -60,6 +64,10 @@ public interface TableBatchOps { void insertOrUpdate(Object[] rowkeys, String[] columns, Object[] values); + void put(Object rowkey, String[] columns, Object[] values); + + void put(Object[] rowkeys, String[] columns, Object[] values); + /** * increment the value * @param rowkey the primary key @@ -102,6 +110,8 @@ public interface TableBatchOps { List execute() throws Exception; + List executeWithResult() throws Exception; + void clear(); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/table/api/TableQuery.java b/src/main/java/com/alipay/oceanbase/rpc/table/api/TableQuery.java index 16bccedb..2f915327 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/table/api/TableQuery.java +++ b/src/main/java/com/alipay/oceanbase/rpc/table/api/TableQuery.java @@ -17,23 +17,39 @@ package com.alipay.oceanbase.rpc.table.api; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; +import com.alipay.oceanbase.rpc.location.model.partition.ObPair; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableEntityType; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHTableFilter; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQuery; import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.table.ObTable; +import com.alipay.oceanbase.rpc.table.ObTableParam; + +import java.util.List; public interface TableQuery { + public static final String TABLE_COMPARE_FILTER = "TableCompareFilter"; ObTableQuery getObTableQuery(); String getTableName(); + TableQuery setRowKey(Row row) throws Exception; + + Row getRowKey() throws Exception; + + List getSelectColumns() throws Exception; + void setEntityType(ObTableEntityType entityType); ObTableEntityType getEntityType(); QueryResultSet execute() throws Exception; + QueryResultSet asyncExecute() throws Exception; + TableQuery select(String... columns); TableQuery setKeys(String... keys); @@ -48,10 +64,12 @@ public interface TableQuery { /** * Row count offset, default: 0 - * + * * @param offset limit offset * @param limit limit count - * @return + * @return this TableQuery + * can not use offset without valid limit value. + * limit lessThan 0 and offset bigThan 0 is not allowed. */ TableQuery limit(int offset, int limit); @@ -60,15 +78,15 @@ public interface TableQuery { /** * Add scan range * - * @param start >= start - * @param end <= end - * @return this + * @param start start + * @param end end + * @return this TableQuery */ TableQuery addScanRange(Object start, Object end); TableQuery addScanRange(Object[] start, Object[] end); - /** + /* * Add scan range * * @param startEquals true: >= start; false: > start @@ -79,7 +97,7 @@ public interface TableQuery { TableQuery addScanRange(Object[] start, boolean startEquals, Object[] end, boolean endEquals); - /** + /* * Add scan range starts with * * @param start >= start @@ -89,16 +107,15 @@ public interface TableQuery { TableQuery addScanRangeStartsWith(Object[] start); - /** + /* * Add scan range starts with - * - * @param startEquals true: >= start; false: > start * @param start >= start + * @param startEquals true: >= start; false: > start * @return this */ TableQuery addScanRangeStartsWith(Object[] start, boolean startEquals); - /** + /* * Add scan range ends with * * @param end <= end @@ -108,15 +125,15 @@ public interface TableQuery { TableQuery addScanRangeEndsWith(Object[] end); - /** + /* * Add scan range ends with - * + * @param end <= end * @param endEquals true: <= end; false: < end * @return this */ TableQuery addScanRangeEndsWith(Object[] end, boolean endEquals); - /** + /* * Scan order, default forward * * @param forward forward(true) or reverse(false) order @@ -124,7 +141,7 @@ public interface TableQuery { */ TableQuery scanOrder(boolean forward); - /** + /* * Set index name * * @param indexName Table index name @@ -142,11 +159,19 @@ public interface TableQuery { /** * Set filter string: no support yet * - * @param filterString + * @param filterString filter * @return this */ TableQuery filterString(String filterString); + /** + * Set filter + * + * @param filter prepared filter + * @return this + */ + TableQuery setFilter(ObTableFilter filter); + TableQuery setHTableFilter(ObHTableFilter obHTableFilter); /** @@ -154,7 +179,7 @@ public interface TableQuery { * default is -1 means one rpc will return all the results * zero or negative value is meaningless so will be reset to default * when user sets the batch size the stream mode will active - * @param batchSize + * @param batchSize batch size * @return this */ TableQuery setBatchSize(int batchSize); @@ -164,11 +189,14 @@ public interface TableQuery { * the default of timeout is 10 second * Be careful about the timeout when you set the batch size ,which should * be completed in query time out - * @param operationTimeout + * @param operationTimeout timeout * @return this */ TableQuery setOperationTimeout(long operationTimeout); - void clear(); + TableQuery setMaxResultSize(long maxResultSize); + + TableQuery setScanRangeColumns(String... columns); + void clear(); } diff --git a/src/main/java/com/alipay/oceanbase/rpc/threadlocal/ThreadLocalMap.java b/src/main/java/com/alipay/oceanbase/rpc/threadlocal/ThreadLocalMap.java index eceb0709..6357e26c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/threadlocal/ThreadLocalMap.java +++ b/src/main/java/com/alipay/oceanbase/rpc/threadlocal/ThreadLocalMap.java @@ -34,28 +34,28 @@ public class ThreadLocalMap { public static final String PROCESS_PRIORITY = "PROCESS_PRIORITY"; public static final String READ_CONSISTENCY = "READ_CONSISTENCY"; - /** + /* * Set process high priority. */ public static void setProcessHighPriority() { getContextMap().put(PROCESS_PRIORITY, (short) 4); } - /** + /* * Set process normal priority. */ public static void setProcessNormalPriority() { getContextMap().put(PROCESS_PRIORITY, (short) 5); } - /** + /* * Set process low priority. */ public static void setProcessLowPriority() { getContextMap().put(PROCESS_PRIORITY, (short) 6); } - /** + /* * Get process priority. */ public static short getProcessPriority() { @@ -67,21 +67,21 @@ public static short getProcessPriority() { return (Short) getContextMap().get(PROCESS_PRIORITY); } - /** + /* * Set read consistency. */ public static void setReadConsistency(ObReadConsistency readConsistency) { getContextMap().put(READ_CONSISTENCY, readConsistency); } - /** + /* * Get read consistency. */ public static ObReadConsistency getReadConsistency() { return (ObReadConsistency) getContextMap().get(READ_CONSISTENCY); } - /** + /* * Clear read consistency. */ public static void clearReadConsistency() { @@ -94,14 +94,14 @@ public static Map getContextMap() { return THREAD_LOCAL_CONTEXT.get(); } - /** + /* * Set context map. */ public static void setContextMap(Map context) { THREAD_LOCAL_CONTEXT.set(context); } - /** + /* * Transmit context map. */ public static void transmitContextMap(Map context) { @@ -116,7 +116,7 @@ protected Map initialValue() { private static final long serialVersionUID = 3637958959138295593L; - /** + /* * put */ public Object put(Object key, Object value) { @@ -136,7 +136,7 @@ public Object put(Object key, Object value) { } } - /** + /* * Reset. */ public static void reset() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/AsyncExecutor.java b/src/main/java/com/alipay/oceanbase/rpc/util/AsyncExecutor.java index 9ca2466d..4e4d438c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/AsyncExecutor.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/AsyncExecutor.java @@ -35,15 +35,18 @@ public class AsyncExecutor { private Exception exception = null; - /** + /* * Async executor. */ public AsyncExecutor() { this(4, 4, 1024); } - /** + /* * Async executor. + * @param corePoolSize the number of threads to keep in the pool + * @param maxPoolSize the maximum number of threads to allow in the pool + * @param queueSize the work queue size */ public AsyncExecutor(int corePoolSize, int maxPoolSize, int queueSize) { executor = new SofaThreadPoolExecutor(corePoolSize, maxPoolSize, 1000, TimeUnit.SECONDS, @@ -51,8 +54,9 @@ public AsyncExecutor(int corePoolSize, int maxPoolSize, int queueSize) { OCEANBASE_TABLE_CLIENT_LOGGER_SPACE); } - /** + /* * Wait complete. + * @throws Exception if something is wrong */ public void waitComplete() throws Exception { startMonitor(); @@ -64,12 +68,13 @@ public void waitComplete() throws Exception { } } - /** + /* * Add task. + * @param runnable task */ public void addTask(final Runnable runnable) { executor.execute(new Runnable() { - /** + /* * Run. */ @Override @@ -88,12 +93,12 @@ public void run() { taskCount.incrementAndGet(); } - /** + /* * monitor executor tasks, if finished, shutdown executor */ private void startMonitor() { new Thread(new Runnable() { - /** + /* * Run. */ @Override diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/Base64.java b/src/main/java/com/alipay/oceanbase/rpc/util/Base64.java index 79ea01b9..11518f40 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/Base64.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/Base64.java @@ -86,7 +86,11 @@ private static int decode0(char[] ibuf, byte[] obuf, int wp) { } /** - * + * Base64 decode + * @param data input data buf + * @param off the data offset + * @param len the data length + * @return plain data */ public static byte[] decode(char[] data, int off, int len) { char[] ibuf = new char[4]; @@ -112,7 +116,9 @@ public static byte[] decode(char[] data, int off, int len) { } /** - * + * Base64 deocde + * @param data input data + * @return plain data */ public static byte[] decode(String data) { char[] ibuf = new char[4]; @@ -138,7 +144,12 @@ public static byte[] decode(String data) { } /** - * + * Base64 decode. + * @param data input data buffer + * @param off input data offset + * @param len input data length + * @param ostream output data stream + * @throws IOException if an I/O error occurs. */ public static void decode(char[] data, int off, int len, OutputStream ostream) throws IOException { @@ -160,7 +171,10 @@ public static void decode(char[] data, int off, int len, OutputStream ostream) } /** - * + * Base64 decode + * @param data input data + * @param ostream output stream + * @throws IOException if an I/O error occurs. */ public static void decode(String data, OutputStream ostream) throws IOException { char[] ibuf = new char[4]; @@ -181,14 +195,20 @@ public static void decode(String data, OutputStream ostream) throws IOException } /** - * Returns base64 representation of specified byte array. + * Base64 encode + * @param data input data + * @return out data */ public static String encode(byte[] data) { return encode(data, 0, data.length); } /** - * Returns base64 representation of specified byte array. + * Base64 encode + * @param data input data + * @param off input data offset + * @param len input data length + * @return output data */ public static String encode(byte[] data, int off, int len) { if (len <= 0) @@ -224,7 +244,12 @@ public static String encode(byte[] data, int off, int len) { } /** - * Outputs base64 representation of the specified byte array to a byte stream. + * Base64 encode + * @param data input data buffer + * @param off input data offset + * @param len input data length + * @param ostream output stream + * @throws IOException if an I/O error occurs. */ public static void encode(byte[] data, int off, int len, OutputStream ostream) throws IOException { @@ -262,7 +287,12 @@ public static void encode(byte[] data, int off, int len, OutputStream ostream) } /** - * Outputs base64 representation of the specified byte array to a character stream. + * Base64 encode + * @param data input data buffer + * @param off input data offset + * @param len input data length + * @param writer output data writer + * @throws IOException if an I/O error occurs. */ public static void encode(byte[] data, int off, int len, Writer writer) throws IOException { if (len <= 0) diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/CRC64.java b/src/main/java/com/alipay/oceanbase/rpc/util/CRC64.java index 03c56a35..2888cf51 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/CRC64.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/CRC64.java @@ -31,7 +31,6 @@ public class CRC64 implements Checksum { * * poly=0x42f0e1eba9ea3693 init=0x0 refin=false refout=false xorout=0x0 * - * @url http://en.wikipedia.org/wiki/Cyclic_redundancy_check * @url http://reveng.sourceforge.net/crc-catalogue/17plus.htm */ private static final long[] CRC_TABLE = new long[] { 0x0000000000000000L, 0x42F0E1EBA9EA3693L, @@ -103,13 +102,15 @@ public class CRC64 implements Checksum { /** * Update. + * @param b input data */ public void update(final int b) { update((byte) (b & 0xFF)); } /** - * Update. + * Update + * @param b input data */ public void update(final byte b) { final int tab_index = ((int) (crc >> 56) ^ b) & 0xFF; @@ -117,7 +118,10 @@ public void update(final byte b) { } /** - * Update. + * Update + * @param buffer input data buffer + * @param offset input data offset + * @param length input data length */ public void update(final byte[] buffer, final int offset, int length) { for (int i = offset; length > 0; length--) @@ -125,7 +129,8 @@ public void update(final byte[] buffer, final int offset, int length) { } /** - * Update. + * Update + * @param buffer input data buffer */ public void update(final byte[] buffer) { for (int i = 0; i < buffer.length; i++) @@ -133,14 +138,15 @@ public void update(final byte[] buffer) { } /** - * Get value. + * Get value + * @return CRC64 value */ public long getValue() { return crc; } /** - * Reset. + * reset */ public void reset() { crc = 0; diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/MonitorUtil.java b/src/main/java/com/alipay/oceanbase/rpc/util/MonitorUtil.java new file mode 100644 index 00000000..ce824d07 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/util/MonitorUtil.java @@ -0,0 +1,316 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.util; + +import com.alibaba.fastjson.JSON; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.*; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateResult; +import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.AbstractQueryStreamResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObNewRange; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQuery; +import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; +import com.alipay.oceanbase.rpc.table.AbstractTableQuery; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static com.alipay.oceanbase.rpc.util.TableClientLoggerFactory.MONITOR; +import static com.alipay.oceanbase.rpc.util.TraceUtil.formatTraceMessage; + +public class MonitorUtil { + private static String buildParamsString(List rowKeys) { + if (rowKeys == null) { + return ""; + } + StringBuilder stringBuilder = new StringBuilder(); + for (Object value : rowKeys) { + if (value instanceof byte[]) { + value = new String((byte[]) value); + } + if (value instanceof ObVString) { + value = ((ObVString) value).getStringVal(); + } + + StringBuilder sb = new StringBuilder(); + String str = sb.append(JSON.toJSON(value)).toString(); + if (str.length() > 10) { + str = str.substring(0, 10); + } + stringBuilder.append(str).append("#"); + } + + return stringBuilder.toString(); + } + + /** + * for QueryAndMutate, i.e. Mutation with Filter + */ + private static String logMessage(String traceId, String database, String tableName, + String methodName, String type, String endpoint, + List params, ObTableQueryAndMutateResult result, + long routeTableTime, long executeTime) { + if (org.apache.commons.lang.StringUtils.isNotBlank(endpoint)) { + endpoint = endpoint.replaceAll(",", "#"); + } + + String argsValue = buildParamsString(params); + + // TODO: Add error no and change the log message + String res = String.valueOf(result.getAffectedRows()); + + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(traceId).append(" - ").append(database).append(",").append(tableName) + .append(",").append(methodName).append(",").append(type).append(",").append(endpoint) + .append(",").append(argsValue).append(",").append(res).append(",") + .append(routeTableTime).append(",").append(executeTime).append(",") + .append(executeTime + routeTableTime); + return stringBuilder.toString(); + } + + public static void info(final ObPayload payload, String database, String tableName, + String methodName, String type, String endpoint, + ObTableQueryAndMutateResult result, ObTableQuery tableQuery, + long routeTableTime, long executeTime, long slowQueryMonitorThreshold) { + if (routeTableTime + executeTime >= slowQueryMonitorThreshold) { + List params = new ArrayList<>(); + for (ObNewRange rang : tableQuery.getKeyRanges()) { + ObRowKey startKey = rang.getStartKey(); + int startKeySize = startKey.getObjs().size(); + ObRowKey endKey = rang.getEndKey(); + int endKeySize = endKey.getObjs().size(); + for (int i = 0; i < startKeySize; i++) { + params.add(startKey.getObj(i).getValue()); + } + + for (int i = 0; i < endKeySize; i++) { + params.add(endKey.getObj(i).getValue()); + } + } + MONITOR.info(logMessage(formatTraceMessage(payload), database, tableName, methodName, + type, endpoint, params, result, routeTableTime, executeTime)); + } + } + + /** + * for table operation, e.g. insert, update + */ + private static String logMessage(String traceId, String database, String tableName, + String methodName, String endpoint, Object[] rowKeys, + ObTableOperationResult result, long routeTableTime, + long executeTime) { + if (org.apache.commons.lang.StringUtils.isNotBlank(endpoint)) { + endpoint = endpoint.replaceAll(",", "#"); + } + // if rowkeys is empty point, then append "rowKeys:null" into log message + String argsValue = (rowKeys == null || rowKeys.length == 0) ? "rowKeys:null" + : buildParamsString(Arrays.asList(rowKeys)); + + ResultCodes resultCode = ResultCodes.valueOf(result.getHeader().getErrno()); + String res = ""; + if (resultCode == ResultCodes.OB_SUCCESS) { + switch (result.getOperationType()) { + case GET: + case INCREMENT: + case APPEND: + res = String.valueOf(result.getEntity().getSimpleProperties().size()); + break; + default: + res = String.valueOf(result.getAffectedRows()); + } + } + String errorCodeStringValue = "UnknownErrorCode"; + if (resultCode != null) { + errorCodeStringValue = resultCode.toString(); + } + + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(traceId).append(" - ").append(database).append(",").append(tableName) + .append(",").append(methodName).append(",").append(endpoint).append(",") + .append(argsValue).append(",").append(errorCodeStringValue).append(",").append(res) + .append(",").append(routeTableTime).append(",").append(executeTime).append(",") + .append(executeTime + routeTableTime); + return stringBuilder.toString(); + } + + public static void info(final ObPayload payload, String database, String tableName, + String methodName, String endpoint, Object[] rowKeys, + ObTableOperationResult result, long routeTableTime, long executeTime, + long slowQueryMonitorThreshold) { + if (routeTableTime + executeTime >= slowQueryMonitorThreshold) { + MONITOR.info(logMessage(formatTraceMessage(payload), database, tableName, methodName, + endpoint, rowKeys, result, routeTableTime, executeTime)); + } + } + + /** + * for batch operation + */ + private static String logMessage(String traceId, String database, String tableName, + String methodName, String endpoint, List rowKeys, + int resultSize, long executeTime) { + if (org.apache.commons.lang.StringUtils.isNotBlank(endpoint)) { + endpoint = endpoint.replaceAll(",", "#"); + } + // if rowkeys is empty point, then append "rowKeys:null" into log message + String argsValue = (rowKeys == null || rowKeys.isEmpty()) ? "rowKeys:null" + : buildParamsString(rowKeys); + + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(traceId).append(" - ").append(database).append(",").append(tableName) + .append(",").append(methodName).append(",").append(endpoint).append(",") + .append(argsValue).append(",").append(resultSize).append(",").append(0).append(",") + .append(executeTime).append(",").append(executeTime); + return stringBuilder.toString(); + } + + // for each sub batch opreation + private static void logMessage0(final ObPayload payload, String database, String tableName, String methodName, String endpoint, ObTableBatchOperation subOperations, + long partId, int resultSize, long executeTime, long slowQueryMonitorThreshold) { + if (executeTime < slowQueryMonitorThreshold) { + return; + } + String traceId = formatTraceMessage(payload); + List ops = subOperations.getTableOperations(); + for (ObTableOperation op : ops) { + List rowKeys = new ArrayList<>(); + ObTableOperationType type = op.getOperationType(); + ObITableEntity entity = op.getEntity(); + if (entity != null) { + long rowkeySize = entity.getRowKeySize(); + ObRowKey rowKey = entity.getRowKey(); + if (rowKey != null && rowkeySize != 0) { + for (int i = 0; i < rowkeySize; i++) { + ObObj obObj = entity.getRowKeyValue(i); + if (obObj != null) { + rowKeys.add(obObj.getValue()); + } + } + } + } + MONITOR.info(logMessage(traceId, database, tableName, methodName+type+"-"+partId, endpoint, rowKeys, resultSize, executeTime)); + } + } + + // for batch operation result + private static String logMessage(String traceId, String database, String tableName, + String methodName, String endpoint, int resultSize, + long routeTableTime, long executeTime) { + if (org.apache.commons.lang.StringUtils.isNotBlank(endpoint)) { + endpoint = endpoint.replaceAll(",", "#"); + } + + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(traceId).append(" - ").append(database).append(",").append(tableName) + .append(",").append(methodName).append(",").append(endpoint).append(",").append(",") + .append(resultSize).append(",").append(routeTableTime).append(",").append(executeTime) + .append(",").append(routeTableTime + executeTime); + return stringBuilder.toString(); + } + + public static void info(final ObPayload payload, String database, String tableName, + String methodName, String endpoint, + ObTableBatchOperation subOperations, long partId, int resultSize, + long executeTime, long slowQueryMonitorThreshold) { + logMessage0(payload, database, tableName, methodName, endpoint, subOperations, partId, + resultSize, executeTime, slowQueryMonitorThreshold); + } + + public static void info(final ObPayload payload, String database, String tableName, + String methodName, String endpoint, int resultSize, + long routeTableTime, long executeTime, long slowQueryMonitorThreshold) { + if (routeTableTime + executeTime >= slowQueryMonitorThreshold) { + MONITOR.info(logMessage(formatTraceMessage(payload), database, tableName, methodName, + endpoint, resultSize, routeTableTime, executeTime)); + } + } + + /** + * for query + */ + private static String logMessage(String traceId, String database, String tableName, + String methodName, String endpoint, List params, + AbstractQueryStreamResult result, long routeTableTime, + long executeTime) { + if (org.apache.commons.lang.StringUtils.isNotBlank(endpoint)) { + endpoint = endpoint.replaceAll(",", "#"); + } + + String argsValue = buildParamsString(params); + + String res = String.valueOf(result.getCacheRows().size()); + + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(traceId).append(",").append(database).append(",").append(tableName) + .append(",").append(methodName).append(",").append(endpoint).append(",") + .append(argsValue).append(",").append(res).append(",").append(routeTableTime) + .append(",").append(executeTime).append(",").append(routeTableTime + executeTime); + return stringBuilder.toString(); + } + + public static void info(final ObPayload payload, String database, String tableName, + String methodName, String endpoint, ObTableQuery tableQuery, + AbstractQueryStreamResult result, long routeTableTime, + long executeTime, long slowQueryMonitorThreshold) { + if (routeTableTime + executeTime >= slowQueryMonitorThreshold) { + List params = new ArrayList<>(); + for (ObNewRange rang : tableQuery.getKeyRanges()) { + ObRowKey startKey = rang.getStartKey(); + int startKeySize = startKey.getObjs().size(); + ObRowKey endKey = rang.getEndKey(); + int endKeySize = endKey.getObjs().size(); + for (int i = 0; i < startKeySize; i++) { + params.add(startKey.getObj(i).getValue()); + } + + for (int i = 0; i < endKeySize; i++) { + params.add(endKey.getObj(i).getValue()); + } + } + + MONITOR.info(logMessage(formatTraceMessage(payload), database, tableName, methodName, endpoint, + params, result, routeTableTime, executeTime)); + } + } + + /** + * for tablet op + */ + private static void logLsOpMessage(final ObPayload payload, String database, String tableName, + String methodName, String endpoint, + ObTableLSOperation lsOperation, int resultSize, + long executeTime, long slowQueryMonitorThreshold) { + if (executeTime < slowQueryMonitorThreshold) { + return; + } + String traceId = formatTraceMessage(payload); + MONITOR.info(logMessage(traceId, database, tableName, + methodName + "-" + lsOperation.getLsId(), endpoint, null, resultSize, executeTime)); + } + + public static void info(final ObPayload payload, String database, String tableName, + String methodName, String endpoint, ObTableLSOperation lsOperation, + int resultSize, long executeTime, long slowQueryMonitorThreshold) { + logLsOpMessage(payload, database, tableName, methodName, endpoint, lsOperation, resultSize, + executeTime, slowQueryMonitorThreshold); + } +} diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/NamePattern.java b/src/main/java/com/alipay/oceanbase/rpc/util/NamePattern.java index edd951da..11665424 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/NamePattern.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/NamePattern.java @@ -23,9 +23,7 @@ /** * NamePattern: patternPrefix_{from1-to1}_{from2-to2}_patternSuffix * - *

* user_{00-15}_suffix, user_{00-15}_{00-11}_suffix - *

* */ public class NamePattern { @@ -49,9 +47,6 @@ public class NamePattern { private boolean hasTwoColumn = false; - /** - * Name pattern. - */ public NamePattern(String patternString) { this.patternString = patternString; String regex = "\\{\\d+-\\d+\\}"; @@ -110,6 +105,8 @@ private void parseSecond(String pattern) { /** * Wrap value. + * @param value value + * @return replace */ public String wrapValue(int value) { int firstValue = value; @@ -119,8 +116,7 @@ public String wrapValue(int value) { secondValue = value % valueRangeSecond; } - // 技术上没必要控制得如此严格 - // 但是为了统一 DB 层库表分布的元数据，需要控制一下 + // 统一 DB 层库表分布的元数据 if (value < minValue || value > maxValue) { throw new IllegalArgumentException("NamePattern value out of bound. value: " + value + ", minValue: " + minValue + ", maxValue: " @@ -139,6 +135,7 @@ public String wrapValue(int value) { /** * Get min value. + * @return min value */ public int getMinValue() { return minValue; @@ -146,6 +143,7 @@ public int getMinValue() { /** * Get max value. + * @return max value */ public int getMaxValue() { return maxValue; @@ -153,6 +151,7 @@ public int getMaxValue() { /** * Get size. + * @return size */ public int getSize() { return maxValue - minValue + 1; diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/NamedThreadFactory.java b/src/main/java/com/alipay/oceanbase/rpc/util/NamedThreadFactory.java index 0454a434..ae54d97c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/NamedThreadFactory.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/NamedThreadFactory.java @@ -22,7 +22,6 @@ /** * Thread factory to name the thread purposely - * */ public class NamedThreadFactory implements ThreadFactory { @@ -41,6 +40,7 @@ public NamedThreadFactory() { /** * Named thread factory. + * @param name thread factory name */ public NamedThreadFactory(String name) { this(name, false); @@ -48,6 +48,8 @@ public NamedThreadFactory(String name) { /** * Named thread factory. + * @param preffix thread name prefix + * @param daemon is daemon */ public NamedThreadFactory(String preffix, boolean daemon) { SecurityManager s = System.getSecurityManager(); @@ -58,8 +60,9 @@ public NamedThreadFactory(String preffix, boolean daemon) { /** * Create a thread. - * * @see java.util.concurrent.ThreadFactory#newThread(java.lang.Runnable) + * @param r runnable task + * @return thread handle */ public Thread newThread(Runnable r) { Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/ObByteBuf.java b/src/main/java/com/alipay/oceanbase/rpc/util/ObByteBuf.java new file mode 100644 index 00000000..696abcb2 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/util/ObByteBuf.java @@ -0,0 +1,92 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.util; + +public class ObByteBuf { + + public byte[] bytes; + public int pos; + + public ObByteBuf(int capacity) { + if (capacity < 0) { + throw new IllegalArgumentException( + String.format("invalid args, capacity(%d)", capacity)); + } + this.bytes = new byte[capacity]; + this.pos = 0; + } + + public ObByteBuf(byte[] bytes) { + if (bytes == null) { + throw new IllegalArgumentException(String.format("invalid args, bytes(null)")); + } + this.bytes = bytes; + this.pos = 0; + } + + /** + * Get capacity + * @return capacity + */ + public int capacity() { + return bytes.length; + } + + /** + * Write byte + * @param b byte + */ + public void writeByte(byte b) { + if (pos + 1 > bytes.length) { + throw new IllegalArgumentException(String.format( + "size overflow, capacity(%d), pos(%d), data_size(1)", bytes.length, pos)); + } + bytes[pos++] = b; + } + + /** + * Write bytes + * @param src bytes + */ + public void writeBytes(byte[] src) { + if (pos + src.length > bytes.length) { + throw new IllegalArgumentException(String.format( + "size overflow, capacity(%d), pos(%d), data_size(%d)", bytes.length, pos, + src.length)); + } + System.arraycopy(src, 0, bytes, pos, src.length); + pos += src.length; + } + + /** + * Write bytes + * @param src data + * @param srcPos start pos + * @param srcLength len + */ + public void writeBytes(byte[] src, int srcPos, int srcLength) { + if (this.pos + srcLength > bytes.length) { + throw new IllegalArgumentException( + String.format("size overflow, capacity(%d), pos(%d), data_size(%d)", bytes.length, + pos, srcLength)); + } + System.arraycopy(src, srcPos, bytes, pos, srcLength); + pos += srcLength; + } + +} \ No newline at end of file diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/ObBytesString.java b/src/main/java/com/alipay/oceanbase/rpc/util/ObBytesString.java index 26d850e0..6619d671 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/ObBytesString.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/ObBytesString.java @@ -17,24 +17,19 @@ package com.alipay.oceanbase.rpc.util; +import java.util.ArrayList; + /** * binary bytes string without charset - * */ public class ObBytesString implements Comparable { public byte[] bytes; - /** - * Ob bytes string. - */ public ObBytesString() { this.bytes = new byte[0]; } - /** - * Ob bytes string. - */ public ObBytesString(byte[] bytes) { if (bytes == null) { throw new IllegalArgumentException("ObBytesString bytes can not be null "); @@ -42,9 +37,6 @@ public ObBytesString(byte[] bytes) { this.bytes = bytes; } - /** - * Ob bytes string. - */ public ObBytesString(String str) { if (str == null) { throw new IllegalArgumentException("ObBytesString str can not be null "); @@ -53,7 +45,8 @@ public ObBytesString(String str) { } /** - * Length. + * Get length + * @return length */ public int length() { return bytes.length; @@ -61,6 +54,8 @@ public int length() { /** * Equals. + * @param o object + * @return equal or not */ @Override public boolean equals(Object o) { @@ -73,7 +68,9 @@ public boolean equals(Object o) { } /** - * Compare to. + * Compare + * @param another byte string + * @return integer greater than, equal to, or less than 0 */ @Override public int compareTo(ObBytesString another) { @@ -95,4 +92,25 @@ private int compare(byte[] s, byte[] t) { } return len1 - len2; } + + public ObBytesString[] split(byte delim) { + ArrayList list = new ArrayList<>(); + int start = 0; + for (int i = 0; i < bytes.length; ++i) { + if (bytes[i] == delim) { + byte[] data = new byte[i - start]; + System.arraycopy(bytes, start, data, 0, data.length); + ObBytesString str = new ObBytesString(data); + list.add(str); + start = i + 1; + } + } + if (start < bytes.length) { + byte[] data = new byte[bytes.length - start]; + System.arraycopy(bytes, start, data, 0, data.length); + ObBytesString str = new ObBytesString(data); + list.add(str); + } + return list.toArray(new ObBytesString[0]); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/ObHashUtils.java b/src/main/java/com/alipay/oceanbase/rpc/util/ObHashUtils.java index c6904e1f..e7b709ca 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/ObHashUtils.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/ObHashUtils.java @@ -17,6 +17,8 @@ package com.alipay.oceanbase.rpc.util; +import com.alipay.oceanbase.rpc.ObGlobal; +import com.alipay.oceanbase.rpc.location.model.partition.ObPartFuncType; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObCollationType; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObColumn; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; @@ -27,14 +29,21 @@ import java.io.UnsupportedEncodingException; import java.sql.Date; import java.sql.Timestamp; +import java.time.OffsetDateTime; import static com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType.*; public class ObHashUtils { /** * Varchar hash. + * @param varchar input varchar data + * @param collationType collation type + * @param hashCode old hashCode + * @param partFuncType partition function type + * @return new hashCode */ - public static long varcharHash(Object varchar, ObCollationType collationType, long hashCode) { + public static long varcharHash(Object varchar, ObCollationType collationType, long hashCode, + ObPartFuncType partFuncType) { // magic number, the same number with observer long seed = 0xc6a4a7935bd1e995L; byte[] bytes; @@ -44,7 +53,8 @@ public static long varcharHash(Object varchar, ObCollationType collationType, lo try { bytes = ((String) varchar).getBytes("UTF-8"); } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("Unsupported Encoding for Object = " + varchar); + throw new IllegalArgumentException("Unsupported Encoding for Object = " + varchar, + e); } } else if (varchar instanceof byte[]) { bytes = (byte[]) varchar; @@ -56,13 +66,31 @@ public static long varcharHash(Object varchar, ObCollationType collationType, lo } switch (collationType) { case CS_TYPE_UTF8MB4_GENERAL_CI: - hashCode = ObHashSortUtf8mb4.obHashSortUtf8Mb4(bytes, bytes.length, hashCode, seed); + if (partFuncType == ObPartFuncType.KEY_V3 + || partFuncType == ObPartFuncType.KEY_IMPLICIT_V2 || ObGlobal.obVsnMajor() >= 4) { + hashCode = ObHashSortUtf8mb4.obHashSortUtf8Mb4(bytes, bytes.length, hashCode, + seed, true); + } else { + hashCode = ObHashSortUtf8mb4.obHashSortUtf8Mb4(bytes, bytes.length, hashCode, + seed, false); + } break; case CS_TYPE_UTF8MB4_BIN: - hashCode = ObHashSortUtf8mb4.obHashSortMbBin(bytes, bytes.length, hashCode, seed); + if (partFuncType == ObPartFuncType.KEY_V3 + || partFuncType == ObPartFuncType.KEY_IMPLICIT_V2 || ObGlobal.obVsnMajor() >= 4) { + hashCode = MurmurHash.hash64a(bytes, bytes.length, hashCode); + } else { + hashCode = ObHashSortUtf8mb4.obHashSortMbBin(bytes, bytes.length, hashCode, + seed); + } break; case CS_TYPE_BINARY: - hashCode = ObHashSortBin.obHashSortBin(bytes, bytes.length, hashCode, seed); + if (partFuncType == ObPartFuncType.KEY_V3 + || partFuncType == ObPartFuncType.KEY_IMPLICIT_V2 || ObGlobal.obVsnMajor() >= 4) { + hashCode = MurmurHash.hash64a(bytes, bytes.length, hashCode); + } else { + hashCode = ObHashSortBin.obHashSortBin(bytes, bytes.length, hashCode, seed); + } break; case CS_TYPE_INVALID: case CS_TYPE_COLLATION_FREE: @@ -76,23 +104,40 @@ public static long varcharHash(Object varchar, ObCollationType collationType, lo } /** - * To hashcode. + * To hash code + * @param value input data + * @param refColumn data info, include type and collation type + * @param hashCode old hashCode + * @param partFuncType partition function type + * @return new hashCode */ - public static long toHashcode(Object value, ObColumn refColumn, long hashCode) { + public static long toHashcode(Object value, ObColumn refColumn, long hashCode, + ObPartFuncType partFuncType) { ObObjType type = refColumn.getObObjType(); int typeValue = type.getValue(); ObCollationType collationType = refColumn.getObCollationType(); if (typeValue >= ObTinyIntType.getValue() && typeValue <= ObUInt64Type.getValue()) { - return ObHashUtils.longHash((Long) value, hashCode); - } else if (ObDateTimeType.getValue() == typeValue - || ObTimestampType.getValue() == typeValue) { + if (value instanceof Integer) { + return ObHashUtils.longHash(((Integer) value).longValue(), hashCode); + } else if (value instanceof Short) { + return ObHashUtils.longHash(((Short) value).longValue(), hashCode); + } else if (value instanceof Byte) { + return ObHashUtils.longHash(((Byte) value).longValue(), hashCode); + } else if (value instanceof Boolean) { + return ObHashUtils.longHash((Boolean) value ? 1L : 0L, hashCode); + } else { + return ObHashUtils.longHash((Long) value, hashCode); + } + } else if (ObTimestampType.getValue() == typeValue) { return ObHashUtils.timeStampHash((Timestamp) value, hashCode); + } else if (ObDateTimeType.getValue() == typeValue) { + return ObHashUtils.dateTimeHash((java.util.Date) value, hashCode); } else if (ObDateType.getValue() == typeValue) { return ObHashUtils.dateHash((Date) value, hashCode); } else if (ObVarcharType.getValue() == typeValue || ObCharType.getValue() == typeValue) { - return ObHashUtils.varcharHash(value, collationType, hashCode); + return ObHashUtils.varcharHash(value, collationType, hashCode, partFuncType); } throw new ClassCastException("unexpected type" + type); @@ -105,21 +150,42 @@ private static byte[] longToByteArray(long l) { } /** - * Long hash. + * Long hash + * @param l input data + * @param hashCode old hashCode + * @return new hashCode */ public static long longHash(long l, long hashCode) { - return MurmurHash.hash64(longToByteArray(l), 8, hashCode); + return MurmurHash.hash64a(longToByteArray(l), 8, hashCode); } /** * Date hash. + * @param d input data + * @param hashCode old hashCode + * @return new hashCode */ public static long dateHash(Date d, long hashCode) { return longHash(d.getTime(), hashCode); } /** - * Time stamp hash. + * Datetime hash (support DateTime(6)). + * @param d input data + * @param hashCode old hashCode + * @return new hashCode + */ + public static long dateTimeHash(java.util.Date d, long hashCode) { + return longHash( + (d.getTime() + OffsetDateTime.now().getOffset().getTotalSeconds() * 1000L) * 1000, + hashCode); + } + + /** + * Time stamp hash + * @param ts input data + * @param hashCode old hashCode + * @return new hashCode */ public static long timeStampHash(Timestamp ts, long hashCode) { return longHash(ts.getTime(), hashCode); diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/ObPureCrc32C.java b/src/main/java/com/alipay/oceanbase/rpc/util/ObPureCrc32C.java index fc4e233b..b7492489 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/ObPureCrc32C.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/ObPureCrc32C.java @@ -85,14 +85,20 @@ public class ObPureCrc32C { 0xad7d5351L }; /** - * Calculate. + * Calculate crc32 + * @param buf input data buffer + * @return CRC32 value */ public static long calculate(byte[] buf) { return calculate(buf, 0, buf.length); } /** - * Calculate. + * Calculate crc32 + * @param buf input data buffer + * @param offset input data offset + * @param length input data length + * @return CRC32 value */ public static long calculate(byte[] buf, int offset, int length) { long crc = 0; diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/ObVString.java b/src/main/java/com/alipay/oceanbase/rpc/util/ObVString.java index 5db0b16f..d7dd16ad 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/ObVString.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/ObVString.java @@ -17,54 +17,37 @@ package com.alipay.oceanbase.rpc.util; -import java.io.UnsupportedEncodingException; - public class ObVString { private String stringVal; private byte[] bytesVal; private byte[] encodeBytes; - /** - * Ob v string. - */ public ObVString(String stringVal) { this.stringVal = stringVal; if (stringVal == null) { bytesVal = new byte[0]; } else { - try { - bytesVal = stringVal.getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("can not encode str to u"); - } + bytesVal = Serialization.strToBytes(stringVal); } - this.encodeBytes = Serialization.encodeVString(stringVal); + this.encodeBytes = Serialization.encodeBytesString(new ObBytesString(bytesVal)); } /** * Get encode need bytes. + * @return return length */ public int getEncodeNeedBytes() { return encodeBytes.length; } - /** - * Get bytes val. - */ public byte[] getBytesVal() { return bytesVal; } - /** - * Get encode bytes. - */ public byte[] getEncodeBytes() { return encodeBytes; } - /** - * Get string val. - */ public String getStringVal() { return stringVal; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/Security.java b/src/main/java/com/alipay/oceanbase/rpc/util/Security.java index 7a1e922b..f9e3c70d 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/Security.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/Security.java @@ -18,22 +18,10 @@ package com.alipay.oceanbase.rpc.util; import com.alipay.oceanbase.rpc.exception.ObTableAuthException; - -import javax.crypto.BadPaddingException; -import javax.crypto.Cipher; -import javax.crypto.IllegalBlockSizeException; -import javax.crypto.NoSuchPaddingException; -import javax.crypto.spec.SecretKeySpec; -import java.math.BigInteger; -import java.security.InvalidKeyException; import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -import static com.alipay.oceanbase.rpc.util.StringUtil.isNotBlank; /** - * {@link com.mysql.jdbc.MysqlIO#secureAuth} - * + * com.mysql.jdbc.MysqlIO#secureAuth */ public class Security { @@ -57,8 +45,9 @@ public class Security { /** * SHA1(passwd) ^ SHA1(scramble + SHA1(SHA1(passwd))) - * - * @return scramble password by scramble salt + * @param password plain password + * @param scrambleSalt salt + * @return cipher password */ public static final ObBytesString scramblePassword(String password, ObBytesString scrambleSalt) { if (password == null || password.isEmpty()) { @@ -90,6 +79,8 @@ public static final ObBytesString scramblePassword(byte[] password, byte[] seed) } /** + * Get password scramble salt + * @param size salt size * @return password scramble salt */ public static final ObBytesString getPasswordScramble(int size) { @@ -115,124 +106,4 @@ private static long random() { seed = nextSeed; return nextSeed; } - - private static String ENC_KEY_BYTES_STR = "jaas is the way"; - private static byte[] ENC_KEY_BYTES = ENC_KEY_BYTES_STR.getBytes(); - - private static String ENC_KEY_BYTES_PROD_STR = "gQzLk5tTcGYlQ47GG29xQxfbHIURCheJ"; - private static byte[] ENC_KEY_BYTES_PROD = ENC_KEY_BYTES_PROD_STR.getBytes(); - - /** - * Encode. - */ - public static String encode(String secret) throws NoSuchPaddingException, - NoSuchAlgorithmException, InvalidKeyException, - BadPaddingException, IllegalBlockSizeException { - return encode(null, secret); - } - - /** - * Encode. - */ - public static String encode(String encKey, String secret) throws InvalidKeyException, - NoSuchAlgorithmException, - NoSuchPaddingException, - IllegalBlockSizeException, - BadPaddingException { - byte[] kbytes = ENC_KEY_BYTES_PROD; - if (isNotBlank(encKey)) { - kbytes = encKey.getBytes(); - } - - // 默认采用prod key加密与解密,线下环境会异常; - try { - return initEncode(kbytes, secret); - } catch (InvalidKeyException e) { - kbytes = ENC_KEY_BYTES; - } catch (NoSuchAlgorithmException e) { - kbytes = ENC_KEY_BYTES; - } catch (NoSuchPaddingException e) { - kbytes = ENC_KEY_BYTES; - } catch (IllegalBlockSizeException e) { - kbytes = ENC_KEY_BYTES; - } catch (BadPaddingException e) { - kbytes = ENC_KEY_BYTES; - } - - return initEncode(kbytes, secret); - } - - static final String initEncode(byte[] kbytes, String secret) throws NoSuchAlgorithmException, - NoSuchPaddingException, - InvalidKeyException, - IllegalBlockSizeException, - BadPaddingException { - SecretKeySpec key = new SecretKeySpec(kbytes, "Blowfish"); - Cipher cipher = Cipher.getInstance("Blowfish"); - cipher.init(Cipher.ENCRYPT_MODE, key); - byte[] encoding = cipher.doFinal(secret.getBytes()); - BigInteger n = new BigInteger(encoding); - return n.toString(16); - } - - /** - * Decode. - */ - public static char[] decode(String secret) throws NoSuchPaddingException, - NoSuchAlgorithmException, InvalidKeyException, - BadPaddingException, IllegalBlockSizeException { - return decode(null, secret).toCharArray(); - } - - /** - * Decode. - */ - public static String decode(String encKey, String secret) throws NoSuchPaddingException, - NoSuchAlgorithmException, - InvalidKeyException, - BadPaddingException, - IllegalBlockSizeException { - - byte[] kbytes = ENC_KEY_BYTES_PROD; - if (isNotBlank(encKey)) { - kbytes = encKey.getBytes(); - } - - try { - return iniDecode(kbytes, secret); - } catch (InvalidKeyException e) { - kbytes = ENC_KEY_BYTES; - } catch (BadPaddingException e) { - kbytes = ENC_KEY_BYTES; - } catch (IllegalBlockSizeException e) { - kbytes = ENC_KEY_BYTES; - } - return iniDecode(kbytes, secret); - } - - private static String iniDecode(byte[] kbytes, String secret) throws NoSuchPaddingException, - NoSuchAlgorithmException, - InvalidKeyException, - BadPaddingException, - IllegalBlockSizeException { - SecretKeySpec secretKeySpec = new SecretKeySpec(kbytes, "Blowfish"); - BigInteger bigInteger = new BigInteger(secret, 16); - byte[] encoding = bigInteger.toByteArray(); - // SECURITY-344: fix leading zeros - if (encoding.length % 8 != 0) { - int len = encoding.length; - int nLen = ((len / 8) + 1) * 8; - int pad = nLen - len; //number of leading zeros - byte[] old = encoding; - encoding = new byte[nLen]; - for (int i = old.length - 1; i >= 0; i--) { - encoding[i + pad] = old[i]; - } - } - Cipher cipher = Cipher.getInstance("Blowfish"); - cipher.init(Cipher.DECRYPT_MODE, secretKeySpec); - byte[] decode = cipher.doFinal(encoding); - return new String(decode); - } - } diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/Serialization.java b/src/main/java/com/alipay/oceanbase/rpc/util/Serialization.java index fd8a3ed7..7b8c0d8e 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/Serialization.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/Serialization.java @@ -17,10 +17,11 @@ package com.alipay.oceanbase.rpc.util; -import com.alipay.oceanbase.rpc.exception.ObTableException; import io.netty.buffer.ByteBuf; import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; public class Serialization { @@ -39,17 +40,11 @@ public class Serialization { /** * bytes length + bytes + 1 - * - * @param val - * @return + * @param val input data + * @return output data */ public static byte[] encodeObString(String val) { - byte[] strbytes = new byte[0]; - try { - strbytes = val.getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new ObTableException(e); - } + byte[] strbytes = strToBytes(val); byte[] length = encodeI64(strbytes.length); byte[] bytes = new byte[length.length + strbytes.length + 1]; @@ -61,29 +56,36 @@ public static byte[] encodeObString(String val) { } /** - * Str to bytes. + * Encode ObString. + * @param buf encoded buf + * @param val input data */ - public static byte[] strToBytes(String str) throws IllegalArgumentException { - try { - return str.getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("can not encode str to bytes with utf-8"); - } + public static void encodeObString(ObByteBuf buf, String val) { + byte[] strBytes = strToBytes(val); + encodeI64(buf, strBytes.length); + buf.writeBytes(strBytes); + buf.writeByte((byte) 0x00); } /** - * Bytes to str. + * Str to bytes. + * @param str input string + * @return output byte array */ - public static String bytesToStr(byte[] bytes) throws IllegalArgumentException { - try { - return new String(bytes, "UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new IllegalArgumentException("can not decode bytes to str with utf-8"); - } + public static byte[] strToBytes(String str) { + if (str == null) + throw new NullPointerException(); + return str.getBytes(StandardCharsets.UTF_8); + } + + public static String bytesToStr(byte[] bytes) { + return new String(bytes, StandardCharsets.UTF_8); } /** * Get ob string serialize size. + * @param val input data + * @return output size */ public static long getObStringSerializeSize(long val) { return encodeLengthVi64(val) + val + 1; @@ -91,6 +93,8 @@ public static long getObStringSerializeSize(long val) { /** * Get ob string serialize size. + * @param val input data + * @return output size */ public static long getObStringSerializeSize(String val) { return getObStringSerializeSize(val.length()); @@ -98,6 +102,8 @@ public static long getObStringSerializeSize(String val) { /** * Encode length vi64. + * @param len data length + * @return bytes need for serialize data length */ public static long encodeLengthVi64(long len) { int needBytes; @@ -127,6 +133,8 @@ public static long encodeLengthVi64(long len) { /** * Encode i8. + * @param val input data + * @return output data */ public static byte[] encodeI8(byte val) { byte[] bytes = new byte[1]; @@ -136,6 +144,19 @@ public static byte[] encodeI8(byte val) { /** * Encode i8. + * @param buf encoded buf + * @param val input data + */ + public static void encodeI8(ObByteBuf buf, byte val) { + if (buf == null) + throw new NullPointerException(); + buf.writeByte((byte) ((val) & 0xff)); + } + + /** + * Encode i8. + * @param val input data + * @return output data */ public static byte[] encodeI8(short val) { byte[] bytes = new byte[1]; @@ -143,8 +164,21 @@ public static byte[] encodeI8(short val) { return bytes; } + /** + * Encode i8. + * @param buf encoded buf + * @param val input data + */ + public static void encodeI8(ObByteBuf buf, short val) { + if (buf == null) + throw new NullPointerException(); + buf.writeByte((byte) ((val) & 0xff)); + } + /** * Encode i16. + * @param val input data + * @return output data */ public static byte[] encodeI16(short val) { byte[] bytes = new byte[4]; @@ -153,8 +187,22 @@ public static byte[] encodeI16(short val) { return bytes; } + /** + * Encode i16. + * @param buf encoded buf + * @param val input data + */ + public static void encodeI16(ObByteBuf buf, short val) { + if (buf == null) + throw new NullPointerException(); + buf.writeByte((byte) ((val >> 8) & 0xff)); + buf.writeByte((byte) ((val) & 0xff)); + } + /** * Encode i32. + * @param val input data + * @return output data */ public static byte[] encodeI32(int val) { byte[] bytes = new byte[4]; @@ -165,8 +213,24 @@ public static byte[] encodeI32(int val) { return bytes; } + /** + * Encode i32. + * @param buf encoded buf + * @param val input data + */ + public static void encodeI32(ObByteBuf buf, int val) { + if (buf == null) + throw new NullPointerException(); + buf.writeByte((byte) ((val >> 24) & 0xff)); + buf.writeByte((byte) ((val >> 16) & 0xff)); + buf.writeByte((byte) ((val >> 8) & 0xff)); + buf.writeByte((byte) ((val) & 0xff)); + } + /** * Encode i64. + * @param val input data + * @return output data */ public static byte[] encodeI64(long val) { byte[] bytes = new byte[8]; @@ -181,15 +245,37 @@ public static byte[] encodeI64(long val) { return bytes; } + /** + * Encode i64. + * @param buf encoded buf + * @param val input data + */ + public static void encodeI64(ObByteBuf buf, long val) { + if (buf == null) + throw new NullPointerException(); + buf.writeByte((byte) ((val >> 56) & 0xff)); + buf.writeByte((byte) ((val >> 48) & 0xff)); + buf.writeByte((byte) ((val >> 40) & 0xff)); + buf.writeByte((byte) ((val >> 32) & 0xff)); + buf.writeByte((byte) ((val >> 24) & 0xff)); + buf.writeByte((byte) ((val >> 16) & 0xff)); + buf.writeByte((byte) ((val >> 8) & 0xff)); + buf.writeByte((byte) ((val) & 0xff)); + } + /** * Decode i8. + * @param val input data buffer + * @return output i8 data. */ public static byte decodeI8(ByteBuf val) { return (byte) (val.readByte() & 0xff); } /** - * Decode u i8. + * Decode UI8. + * @param val input data buffer + * @return output UI8 data. */ public static short decodeUI8(ByteBuf val) { return (short) (val.readUnsignedByte() & 0xffff); @@ -197,6 +283,8 @@ public static short decodeUI8(ByteBuf val) { /** * Decode i8. + * @param val input data + * @return output i8 data. */ public static byte decodeI8(byte val) { return (byte) (val & 0xff); @@ -204,6 +292,8 @@ public static byte decodeI8(byte val) { /** * Decode i8. + * @param val input data buffer + * @return output i8 data. */ public static byte decodeI8(byte[] val) { return (byte) (val[0] & 0xff); @@ -211,6 +301,8 @@ public static byte decodeI8(byte[] val) { /** * Decode i16. + * @param val input data + * @return output i16 data. */ public static short decodeI16(ByteBuf val) { short ret = (short) ((val.readByte() & 0xff) << 8); @@ -220,6 +312,8 @@ public static short decodeI16(ByteBuf val) { /** * Decode i16. + * @param val input data buffer + * @return output i16 data. */ public static short decodeI16(byte[] val) { short ret = (short) ((val[0] & 0xff) << 8); @@ -229,6 +323,8 @@ public static short decodeI16(byte[] val) { /** * Decode i32. + * @param val input data buffer + * @return output i32 data. */ public static int decodeI32(byte[] val) { int ret = (val[0] & 0xff) << 24; @@ -240,6 +336,8 @@ public static int decodeI32(byte[] val) { /** * Decode i32. + * @param val input data + * @return output i32 data. */ public static int decodeI32(ByteBuf val) { int ret = (val.readByte() & 0xff) << 24; @@ -251,6 +349,8 @@ public static int decodeI32(ByteBuf val) { /** * Decode i64. + * @param val input data buffer + * @return output i64 data. */ public static long decodeI64(byte[] val) { long ret = ((long) (val[0] & 0xff)) << 56; @@ -266,6 +366,8 @@ public static long decodeI64(byte[] val) { /** * Decode i64. + * @param val input data + * @return output i64 data. */ public static long decodeI64(ByteBuf val) { long ret = ((long) (val.readByte() & 0xff)) << 56; @@ -281,6 +383,8 @@ public static long decodeI64(ByteBuf val) { /** * Decode double. + * @param buffer input data buffer + * @return output double data. */ public static double decodeDouble(ByteBuf buffer) { return Double.longBitsToDouble(Serialization.decodeVi64(buffer)); @@ -288,13 +392,26 @@ public static double decodeDouble(ByteBuf buffer) { /** * Encode double. + * @param d input double data + * @return output data buffer */ public static byte[] encodeDouble(double d) { return Serialization.encodeVi64(Double.doubleToRawLongBits(d)); } + /** + * Encode double. + * @param buf encoded buf + * @param d input data + */ + public static void encodeDouble(ObByteBuf buf, double d) { + encodeVi64(buf, Double.doubleToRawLongBits(d)); + } + /** * Decode float. + * @param value input data buffer + * @return output float data. */ public static float decodeFloat(ByteBuf value) { return Float.intBitsToFloat(decodeVi32(value)); @@ -302,13 +419,26 @@ public static float decodeFloat(ByteBuf value) { /** * Encode float. + * @param f input float data + * @return output data buffer */ public static byte[] encodeFloat(float f) { return encodeVi32(Float.floatToRawIntBits(f)); } + /** + * Encode float. + * @param buf encoded buf + * @param f input data + */ + public static void encodeFloat(ObByteBuf buf, float f) { + encodeVi32(buf, Float.floatToRawIntBits(f)); + } + /** * Get need bytes. + * @param l input data + * @return bytes need for serialize long data */ public static int getNeedBytes(long l) { if (l < 0) @@ -324,6 +454,8 @@ public static int getNeedBytes(long l) { /** * Get need bytes. + * @param l input data + * @return bytes need for serialize int data */ public static int getNeedBytes(int l) { if (l < 0) @@ -339,6 +471,8 @@ public static int getNeedBytes(int l) { /** * Get need bytes. + * @param str input data + * @return bytes need for serialize string data */ public static int getNeedBytes(ObBytesString str) { if (str == null) { @@ -349,6 +483,8 @@ public static int getNeedBytes(ObBytesString str) { /** * Get need bytes. + * @param val input data + * @return bytes need for serialize double data */ public static int getNeedBytes(double val) { return getNeedBytes(Double.doubleToRawLongBits(val)); @@ -356,6 +492,8 @@ public static int getNeedBytes(double val) { /** * Get need bytes. + * @param val input data + * @return bytes need for serialize float data */ public static int getNeedBytes(float val) { return getNeedBytes(Float.floatToRawIntBits(val)); @@ -363,6 +501,8 @@ public static int getNeedBytes(float val) { /** * Get need bytes. + * @param bytes input data + * @return bytes need for serialize byte array data */ public static int getNeedBytes(byte[] bytes) { if (bytes == null) { @@ -373,20 +513,37 @@ public static int getNeedBytes(byte[] bytes) { /** * Get need bytes. + * @param str input data + * @return bytes need for serialize string data */ public static int getNeedBytes(String str) { if (str == null) str = ""; - try { - int len = str.getBytes("UTF-8").length; - return getNeedBytes(len) + len + 1; - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); + int utf8Length = 0; + for (int i = 0; i < str.length();) { + int codePoint = str.codePointAt(i); + utf8Length += getUtf8Length(codePoint); + i += Character.charCount(codePoint); + } + return getNeedBytes(utf8Length) + utf8Length + 1; + } + + private static int getUtf8Length(int codePoint) { + if (codePoint <= 0x7F) { + return 1; + } else if (codePoint <= 0x7FF) { + return 2; + } else if (codePoint <= 0xFFFF) { + return 3; + } else { + return 4; } } /** - * Encode vi32. + * Encode vi32 + * @param i input data + * @return output data buffer */ public static byte[] encodeVi32(int i) { byte[] ret = new byte[getNeedBytes(i)]; @@ -399,8 +556,25 @@ public static byte[] encodeVi32(int i) { return ret; } + /** + * Encode vi32. + * @param buf encoded buf + * @param i input data + */ + public static void encodeVi32(ObByteBuf buf, int i) { + if (buf == null) + throw new NullPointerException(); + while (i < 0 || i > OB_MAX_V1B) { + buf.writeByte((byte) (i | 0x80)); + i >>>= 7; + } + buf.writeByte((byte) (i & 0x7f)); + } + /** * Decode vi32. + * @param value input data buffer + * @return output int data */ public static int decodeVi32(byte[] value) { int ret = 0; @@ -417,6 +591,8 @@ public static int decodeVi32(byte[] value) { /** * Decode vi32. + * @param buffer input data buffer + * @return output int data */ public static int decodeVi32(ByteBuf buffer) { int ret = 0; @@ -434,6 +610,8 @@ public static int decodeVi32(ByteBuf buffer) { /** * Decode vi64. + * @param buffer input data buffer + * @return output long data */ public static long decodeVi64(ByteBuf buffer) { long ret = 0; @@ -450,7 +628,9 @@ public static long decodeVi64(ByteBuf buffer) { } /** - * Encode vi64. + * Encode vi64 + * @param l input data + * @return output data buffer */ public static byte[] encodeVi64(long l) { byte[] ret = new byte[getNeedBytes(l)]; @@ -463,8 +643,25 @@ public static byte[] encodeVi64(long l) { return ret; } + /** + * Encode vi64. + * @param buf encoded buf + * @param l input data + */ + public static void encodeVi64(ObByteBuf buf, long l) { + if (buf == null) + throw new NullPointerException(); + while (l < 0 || l > OB_MAX_V1B) { + buf.writeByte((byte) (l | 0x80)); + l >>>= 7; + } + buf.writeByte((byte) (l & 0x7f)); + } + /** * Decode vi64. + * @param value input data buffer + * @return output long data */ public static long decodeVi64(byte[] value) { long ret = 0; @@ -480,14 +677,27 @@ public static long decodeVi64(byte[] value) { } /** - * Encode v string. + * Encode VString + * @param str input data + * @return output data buffer */ public static byte[] encodeVString(String str) { - return encodeVString(str, "UTF-8"); + return encodeVString(str, StandardCharsets.UTF_8); + } + + /** + * Encode VString. + * @param buf encoded buf + * @param str input data + */ + public static void encodeVString(ObByteBuf buf, String str) { + encodeVString(buf, str, StandardCharsets.UTF_8); } /** - * Encode bytes string. + * Encode BytesString + * @param str input data + * @return output data buffer */ public static byte[] encodeBytesString(ObBytesString str) { if (str == null) { @@ -509,7 +719,25 @@ public static byte[] encodeBytesString(ObBytesString str) { } /** - * Encode bytes. + * Encode BytesString. + * @param buf encoded buf + * @param str input data + */ + public static void encodeBytesString(ObByteBuf buf, ObBytesString str) { + if (buf == null) + throw new NullPointerException(); + int dataLen = (str == null ? 0 : str.length()); + encodeVi32(buf, dataLen); + if (str != null) { + buf.writeBytes(str.bytes); + } + buf.writeByte((byte) 0x00); + } + + /** + * Encode bytes + * @param bytes input data + * @return output data buffer */ public static byte[] encodeBytes(byte[] bytes) { if (bytes == null) { @@ -530,7 +758,25 @@ public static byte[] encodeBytes(byte[] bytes) { } /** - * Encode v string. + * Encode bytes + * @param buf encoded buf + * @param bytes input data + */ + public static void encodeBytes(ObByteBuf buf, byte[] bytes) { + if (buf == null) + throw new NullPointerException(); + int bytesLen = (bytes != null ? bytes.length : 0); + encodeVi32(buf, bytesLen); + if (bytes != null) { + buf.writeBytes(bytes); + } + } + + /** + * Encode VString + * @param str input data + * @param charset input data charset + * @return output data buffer */ public static byte[] encodeVString(String str, String charset) { if (str == null) @@ -554,8 +800,43 @@ public static byte[] encodeVString(String str, String charset) { } } + public static byte[] encodeVString(String str, Charset charset) { + if (str == null) + str = ""; + byte[] data = str.getBytes(charset); + int dataLen = data.length; + int strLen = getNeedBytes(dataLen); + byte[] ret = new byte[strLen + dataLen + 1]; + int index = 0; + for (byte b : encodeVi32(dataLen)) { + ret[index++] = b; + } + for (byte b : data) { + ret[index++] = b; + } + ret[index] = 0; + return ret; + } + + /** + * Encode VString + * @param buf encoded buf + * @param str input data + * @param charset input data charset + */ + public static void encodeVString(ObByteBuf buf, String str, Charset charset) { + if (buf == null || str == null) + throw new NullPointerException(); + byte[] strBytes = str.getBytes(charset); + encodeVi32(buf, strBytes.length); + buf.writeBytes(strBytes); + buf.writeByte((byte) 0x00); + } + /** - * Decode bytes string. + * Decode byte string + * @param buf input data + * @return output data */ public static ObBytesString decodeBytesString(ByteBuf buf) { int dataLen = decodeVi32(buf); @@ -566,7 +847,9 @@ public static ObBytesString decodeBytesString(ByteBuf buf) { } /** - * Decode bytes. + * Decode bytes + * @param buf input data + * @return output data */ public static byte[] decodeBytes(ByteBuf buf) { int dataLen = decodeVi32(buf); @@ -576,14 +859,19 @@ public static byte[] decodeBytes(ByteBuf buf) { } /** - * Decode v string. + * Decode VString + * @param buf input data + * @return output data */ public static String decodeVString(ByteBuf buf) { - return decodeVString(buf, "UTF-8"); + return decodeVString(buf, StandardCharsets.UTF_8); } /** - * Decode v string. + * Decode byte string + * @param buffer input data + * @param charset input data charset + * @return output data */ public static String decodeVString(ByteBuf buffer, String charset) { int dataLen = decodeVi32(buffer); @@ -593,15 +881,28 @@ public static String decodeVString(ByteBuf buffer, String charset) { return decodeVString(content, charset); } + public static String decodeVString(ByteBuf buffer, Charset charset) { + int dataLen = decodeVi32(buffer); + byte[] content = new byte[dataLen]; + buffer.readBytes(content); + buffer.readByte();// skip the end byte '0' + return decodeVString(content, charset); + } + /** - * Decode v string. + * Decode VString + * @param content input data + * @return output data */ public static String decodeVString(byte[] content) { - return decodeVString(content, "UTF-8"); + return decodeVString(content, StandardCharsets.UTF_8); } /** - * Decode v string. + * Decode byte string + * @param content input data + * @param charset input data charset + * @return output data */ public static String decodeVString(byte[] content, String charset) { try { @@ -611,8 +912,15 @@ public static String decodeVString(byte[] content, String charset) { } } + public static String decodeVString(byte[] content, Charset charset) { + return new String(content, charset); + } + /** * Get ob uni version header length. + * @param version version + * @param payloadLen paylaod length + * @return bytes need for version and payloadLen */ public static long getObUniVersionHeaderLength(long version, long payloadLen) { return Serialization.getNeedBytes(version) + Serialization.getNeedBytes(payloadLen); @@ -620,6 +928,9 @@ public static long getObUniVersionHeaderLength(long version, long payloadLen) { /** * Encode ob uni version header. + * @param version version + * @param payloadLen payload length + * @return output data buffer */ public static byte[] encodeObUniVersionHeader(long version, long payloadLen) { byte[] bytes = new byte[(int) getObUniVersionHeaderLength(version, payloadLen)]; @@ -634,4 +945,14 @@ public static byte[] encodeObUniVersionHeader(long version, long payloadLen) { return bytes; } + /** + * Encode ob uni version header. + * @param buf encoded buf + * @param version version + * @param payloadLen payload length + */ + public static void encodeObUniVersionHeader(ObByteBuf buf, long version, long payloadLen) { + encodeVi64(buf, version); + encodeVi64(buf, payloadLen); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/StringUtil.java b/src/main/java/com/alipay/oceanbase/rpc/util/StringUtil.java index 8fe4618f..4b6c4669 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/StringUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/StringUtil.java @@ -19,16 +19,10 @@ public class StringUtil { - /** - * Is empty. - */ public static boolean isEmpty(String str) { return str == null || str.isEmpty(); } - /** - * Is not empty. - */ public static boolean isNotEmpty(String str) { return !isEmpty(str); } @@ -136,7 +130,6 @@ public static String alignRight(String str, int size, String padStr) { /** * 替换指定的子串，只替换第一个出现的子串。 * - *

* 如果字符串为null则返回null，如果指定子串为null，则返回原字符串。 * 

      * StringUtil.replaceOnce(null, *, *)        = null
@@ -147,7 +140,6 @@ public static String alignRight(String str, int size, String padStr) {
      * StringUtil.replaceOnce("aba", "a", "")    = "ba"
      * StringUtil.replaceOnce("aba", "a", "z")   = "zba"
      *
- *

* * @param text 要扫描的字符串 * @param repl 要搜索的子串 @@ -162,7 +154,6 @@ public static String replaceOnce(String text, String repl, String with) { /** * 替换指定的子串，替换指定的次数。 * - *

* 如果字符串为null则返回null，如果指定子串为null，则返回原字符串。 * 

      * StringUtil.replace(null, *, *, *)         = null
@@ -176,7 +167,6 @@ public static String replaceOnce(String text, String repl, String with) {
      * StringUtil.replace("abaa", "a", "z", 2)   = "zbza"
      * StringUtil.replace("abaa", "a", "z", -1)  = "zbzz"
      *
- *

* * @param text 要扫描的字符串 * @param repl 要搜索的子串 @@ -210,6 +200,8 @@ public static String replace(String text, String repl, String with, int max) { /** * Parse name pattern. + * @param pattern pattern + * @return string array */ public static String[] parseNamePattern(String pattern) { String dbIndexes[]; diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/TableClientLoggerFactory.java b/src/main/java/com/alipay/oceanbase/rpc/util/TableClientLoggerFactory.java index 576a4a16..339da27c 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/TableClientLoggerFactory.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/TableClientLoggerFactory.java @@ -18,21 +18,44 @@ package com.alipay.oceanbase.rpc.util; import com.alipay.sofa.common.code.LogCode2Description; +import com.alipay.sofa.common.log.MultiAppLoggerSpaceManager; +import com.alipay.sofa.common.log.adapter.level.AdapterLevel; import org.slf4j.Logger; -import com.alipay.sofa.common.log.LoggerSpaceManager; +import org.slf4j.helpers.NOPLogger; + +import java.util.Map; public class TableClientLoggerFactory { public static final String OCEANBASE_TABLE_CLIENT_LOGGER_SPACE = "oceanbase-table-client"; + public static final String OCEANBASE_TABLE_CLIENT_BOOT = "OBKV-BOOT"; + public static final String OCEANBASE_TABLE_CLIENT_MONITOR = "OBKV-MONITOR"; + public static final String OCEANBASE_TABLE_CLIENT_RUNTIME = "OBKV-RUNTIME"; public static LogCode2Description LCD = LogCode2Description .create(OCEANBASE_TABLE_CLIENT_LOGGER_SPACE); + public static Logger BOOT = NOPLogger.NOP_LOGGER; + public static Logger MONITOR = NOPLogger.NOP_LOGGER; + public static Logger RUNTIME = NOPLogger.NOP_LOGGER; + + static { + BOOT = getBootLogger(); + MONITOR = getMonitorLogger(); + RUNTIME = getRUNTIMELogger(); + } + public static Logger getLogger(String name) { if (name == null || name.isEmpty()) { return null; } - return LoggerSpaceManager.getLoggerBySpace(name, OCEANBASE_TABLE_CLIENT_LOGGER_SPACE); + if (!MultiAppLoggerSpaceManager.isSpaceInitialized(OCEANBASE_TABLE_CLIENT_LOGGER_SPACE)) { + //initLogger(); + MultiAppLoggerSpaceManager.init(OCEANBASE_TABLE_CLIENT_LOGGER_SPACE, null); + } + + return MultiAppLoggerSpaceManager.getLoggerBySpace(name, + OCEANBASE_TABLE_CLIENT_LOGGER_SPACE); } public static Logger getLogger(Class klass) { @@ -42,4 +65,39 @@ public static Logger getLogger(Class klass) { return getLogger(klass.getCanonicalName()); } + + public static Logger getBootLogger() { + if (BOOT == NOPLogger.NOP_LOGGER) { + BOOT = new WrappedLogger(getLogger(OCEANBASE_TABLE_CLIENT_BOOT)); + } + + return BOOT; + } + + public static Logger getMonitorLogger() { + if (MONITOR == NOPLogger.NOP_LOGGER) { + MONITOR = new WrappedLogger(getLogger(OCEANBASE_TABLE_CLIENT_MONITOR)); + } + + return MONITOR; + } + + public static Logger getRUNTIMELogger() { + if (RUNTIME == NOPLogger.NOP_LOGGER) { + RUNTIME = new WrappedLogger(getLogger(OCEANBASE_TABLE_CLIENT_RUNTIME)); + } + + return RUNTIME; + } + + public static void changeLevel(Map levelMap) { + for (Map.Entry entry : levelMap.entrySet()) { + changeLevel(entry.getKey(), entry.getValue()); + } + } + + public static void changeLevel(String name, String level) { + MultiAppLoggerSpaceManager.setLoggerLevel(name, OCEANBASE_TABLE_CLIENT_LOGGER_SPACE, + AdapterLevel.getAdapterLevel(level)); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/TimeUtils.java b/src/main/java/com/alipay/oceanbase/rpc/util/TimeUtils.java index 152158d1..d1f16c75 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/TimeUtils.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/TimeUtils.java @@ -36,7 +36,7 @@ public static long getCurrentTimeMs() { return System.currentTimeMillis(); } - /** + /* * Get current time us. */ public static long getCurrentTimeUs() { @@ -49,14 +49,14 @@ public static long getNanoTimeUs() { return System.nanoTime() / 1000; } - /** + /* * Get nano time ns. */ public static long getNanoTimeNs() { return System.nanoTime(); } - /** + /* * Sleep. */ public static void sleep(long timeMs) { @@ -67,7 +67,7 @@ public static void sleep(long timeMs) { } } - /** + /* * Format time us to date. */ public static String formatTimeUsToDate(long timeUS) { @@ -77,7 +77,7 @@ public static String formatTimeUsToDate(long timeUS) { return sdf.format(c.getTime()); } - /** + /* * Format time ms to date. */ public static String formatTimeMsToDate(long timeMS) { @@ -87,7 +87,7 @@ public static String formatTimeMsToDate(long timeMS) { return sdf.format(c.getTime()); } - /** + /* * Get time. */ public static String getTime(String format) { @@ -96,7 +96,7 @@ public static String getTime(String format) { return sdf.format(c.getTime()); } - /** + /* * Str to timestamp. */ public static Timestamp strToTimestamp(String str) { @@ -117,7 +117,7 @@ public static Timestamp strToTimestamp(String str) { return ts; } - /** + /* * Str to date. */ public static Date strToDate(String str) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/TraceUtil.java b/src/main/java/com/alipay/oceanbase/rpc/util/TraceUtil.java index 81379ac5..e4ecb532 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/TraceUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/TraceUtil.java @@ -24,30 +24,30 @@ public class TraceUtil { - /** + /* * Trace format from connection and payload. */ public static String formatTraceMessage(final ObTableConnection conn, final ObPayload payload) { return formatTraceMessage(conn, payload, ""); } - /** + /* * Trace format from connection and payload. */ public static String formatTraceMessage(final ObTableConnection conn, final ObPayload payload, final String msg) { - return String.format("[Y%X-%016X] server [%s:%d] %s", payload.getSequence(), - payload.getUniqueId(), conn.getObTable().getIp(), conn.getObTable().getPort(), msg); + return String.format("[Y%X-%016X] server [%s:%d] %s", payload.getUniqueId(), + payload.getSequence(), conn.getObTable().getIp(), conn.getObTable().getPort(), msg); } - /** + /* * Trace format from connection and packet, used when the packet isn't decoded yet. */ public static String formatTraceMessage(final ObTableConnection conn, final ObTablePacket packet) { return formatTraceMessage(conn, packet, ""); } - /** + /* * Trace format from connection and packet, used when the packet isn't decoded yet. */ public static String formatTraceMessage(final ObTableConnection conn, @@ -57,10 +57,14 @@ public static String formatTraceMessage(final ObTableConnection conn, .getPort(), msg); } - /** + /* * Format IP:port from ObTable. */ public static String formatIpPort(final ObTable obTable) { return String.format("server [%s:%d]", obTable.getIp(), obTable.getPort()); } + + public static String formatTraceMessage(final ObPayload payload) { + return String.format("[Y%X-%016X]", payload.getUniqueId(), payload.getSequence()); + } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/WrappedLogger.java b/src/main/java/com/alipay/oceanbase/rpc/util/WrappedLogger.java new file mode 100644 index 00000000..9b46864d --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/rpc/util/WrappedLogger.java @@ -0,0 +1,374 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2022 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.util; + +import org.slf4j.Logger; +import org.slf4j.Marker; +import com.alipay.remoting.util.StringUtils; + +/** + *

Date: 2016-08-17 Time: 10:22

+ * + * @author jiyong.jy + */ +public class WrappedLogger implements Logger { + + private Logger logger; + + public WrappedLogger(Logger logger) { + this.logger = logger; + } + + private String getLoggerString(String s) { + StringBuilder stringBuilder = new StringBuilder(); + String traceId = getTraceId(); + if (StringUtils.isNotBlank(traceId)) { + stringBuilder.append(traceId).append("-"); + } + stringBuilder.append(s); + return stringBuilder.toString(); + } + + private String getTraceId() { + String tracerId = ""; + return tracerId; + } + + @Override + public String getName() { + return logger.getName(); + } + + @Override + public boolean isTraceEnabled() { + return logger.isTraceEnabled(); + } + + @Override + public void trace(String s) { + logger.trace(getLoggerString(s)); + } + + @Override + public void trace(String s, Object o) { + logger.trace(getLoggerString(s), o); + } + + @Override + public void trace(String s, Object o, Object o1) { + logger.trace(getLoggerString(s), o, o1); + } + + @Override + public void trace(String s, Object[] objects) { + logger.trace(getLoggerString(s), objects); + } + + @Override + public void trace(String s, Throwable throwable) { + logger.trace(getLoggerString(s), throwable); + } + + @Override + public boolean isTraceEnabled(Marker marker) { + return logger.isTraceEnabled(marker); + } + + @Override + public void trace(Marker marker, String s) { + logger.trace(marker, s); + } + + @Override + public void trace(Marker marker, String s, Object o) { + logger.trace(marker, s, o); + } + + @Override + public void trace(Marker marker, String s, Object o, Object o1) { + logger.trace(marker, s, o, o1); + } + + @Override + public void trace(Marker marker, String s, Object[] objects) { + logger.trace(marker, s, objects); + } + + @Override + public void trace(Marker marker, String s, Throwable throwable) { + logger.trace(marker, s, throwable); + } + + @Override + public boolean isDebugEnabled() { + return logger.isDebugEnabled(); + } + + @Override + public void debug(String s) { + logger.debug(getLoggerString(s)); + } + + @Override + public void debug(String s, Object o) { + if (logger.isDebugEnabled()) { + logger.debug(getLoggerString(s), o); + } + } + + @Override + public void debug(String s, Object o, Object o1) { + if (logger.isDebugEnabled()) { + logger.debug(getLoggerString(s), o, o1); + } + } + + @Override + public void debug(String s, Object[] objects) { + if (logger.isDebugEnabled()) { + logger.debug(getLoggerString(s), objects); + } + } + + @Override + public void debug(String s, Throwable throwable) { + if (logger.isDebugEnabled()) { + logger.debug(getLoggerString(s), throwable); + } + } + + @Override + public boolean isDebugEnabled(Marker marker) { + return logger.isDebugEnabled(marker); + } + + @Override + public void debug(Marker marker, String s) { + logger.debug(marker, s); + } + + @Override + public void debug(Marker marker, String s, Object o) { + logger.debug(marker, s, o); + } + + @Override + public void debug(Marker marker, String s, Object o, Object o1) { + logger.debug(marker, s, o, o1); + } + + @Override + public void debug(Marker marker, String s, Object[] objects) { + logger.debug(marker, s, objects); + } + + @Override + public void debug(Marker marker, String s, Throwable throwable) { + logger.debug(marker, s, throwable); + } + + @Override + public boolean isInfoEnabled() { + return logger.isInfoEnabled(); + } + + @Override + public void info(String s) { + if (logger.isInfoEnabled()) { + logger.info(getLoggerString(s)); + } + } + + @Override + public void info(String s, Object o) { + if (logger.isInfoEnabled()) { + logger.info(getLoggerString(s), o); + } + } + + @Override + public void info(String s, Object o, Object o1) { + if (logger.isInfoEnabled()) { + logger.info(getLoggerString(s), o, o1); + } + } + + @Override + public void info(String s, Object[] objects) { + if (logger.isInfoEnabled()) { + logger.info(getLoggerString(s), objects); + } + } + + @Override + public void info(String s, Throwable throwable) { + if (logger.isInfoEnabled()) { + logger.info(getLoggerString(s), throwable); + } + } + + @Override + public boolean isInfoEnabled(Marker marker) { + return logger.isInfoEnabled(marker); + } + + @Override + public void info(Marker marker, String s) { + logger.info(marker, s); + } + + @Override + public void info(Marker marker, String s, Object o) { + logger.info(marker, s, o); + } + + @Override + public void info(Marker marker, String s, Object o, Object o1) { + logger.info(marker, s, o, o1); + } + + @Override + public void info(Marker marker, String s, Object[] objects) { + logger.info(marker, s, objects); + } + + @Override + public void info(Marker marker, String s, Throwable throwable) { + logger.info(marker, s, throwable); + } + + @Override + public boolean isWarnEnabled() { + return logger.isWarnEnabled(); + } + + @Override + public void warn(String s) { + logger.warn(getLoggerString(s)); + } + + @Override + public void warn(String s, Object o) { + logger.warn(getLoggerString(s), o); + } + + @Override + public void warn(String s, Object[] objects) { + logger.warn(getLoggerString(s), objects); + } + + @Override + public void warn(String s, Object o, Object o1) { + logger.warn(getLoggerString(s), o, o1); + } + + @Override + public void warn(String s, Throwable throwable) { + logger.warn(getLoggerString(s), throwable); + } + + @Override + public boolean isWarnEnabled(Marker marker) { + return logger.isWarnEnabled(marker); + } + + @Override + public void warn(Marker marker, String s) { + logger.warn(marker, s); + } + + @Override + public void warn(Marker marker, String s, Object o) { + logger.warn(marker, s, o); + } + + @Override + public void warn(Marker marker, String s, Object o, Object o1) { + logger.warn(marker, s, o, o1); + } + + @Override + public void warn(Marker marker, String s, Object[] objects) { + logger.warn(marker, s, objects); + } + + @Override + public void warn(Marker marker, String s, Throwable throwable) { + logger.warn(marker, s, throwable); + } + + @Override + public boolean isErrorEnabled() { + return logger.isErrorEnabled(); + } + + @Override + public void error(String s) { + logger.error(getLoggerString(s)); + } + + @Override + public void error(String s, Object o) { + logger.error(getLoggerString(s), o); + } + + @Override + public void error(String s, Object o, Object o1) { + logger.error(getLoggerString(s), o, o1); + } + + @Override + public void error(String s, Object[] objects) { + logger.error(getLoggerString(s), objects); + } + + @Override + public void error(String s, Throwable throwable) { + logger.error(getLoggerString(s), throwable); + } + + @Override + public boolean isErrorEnabled(Marker marker) { + return logger.isErrorEnabled(marker); + } + + @Override + public void error(Marker marker, String s) { + logger.error(marker, s); + } + + @Override + public void error(Marker marker, String s, Object o) { + logger.error(marker, s, o); + } + + @Override + public void error(Marker marker, String s, Object o, Object o1) { + logger.error(marker, s, o, o1); + } + + @Override + public void error(Marker marker, String s, Object[] objects) { + logger.error(marker, s, objects); + } + + @Override + public void error(Marker marker, String s, Throwable throwable) { + logger.error(marker, s, throwable); + } +} \ No newline at end of file diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/YcsbBench.java b/src/main/java/com/alipay/oceanbase/rpc/util/YcsbBench.java deleted file mode 100644 index 4d5aba08..00000000 --- a/src/main/java/com/alipay/oceanbase/rpc/util/YcsbBench.java +++ /dev/null @@ -1,148 +0,0 @@ -/*- - * #%L - * OBKV Table Client Framework - * %% - * Copyright (C) 2021 OceanBase - * %% - * OBKV Table Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.rpc.util; - -import com.alipay.oceanbase.rpc.table.ObTable; -import com.yahoo.ycsb.*; - -import java.util.*; - -public class YcsbBench extends DB { - - public static final String HOST_PROPERTY = "ob.table.host"; - public static final String PORT_PROPERTY = "ob.table.port"; - public static final String TENANT_PROPERTY = "ob.table.tenant"; - public static final String USERNAME_PROPERTY = "ob.table.username"; - public static final String PASSWORD_PROPERTY = "ob.table.password"; - public static final String DB_PROPERTY = "ob.table.database"; - - ObTable client; - String[] fields = new String[] { "FIELD0", "FIELD1", "FIELD2", - "FIELD3", "FIELD4", "FIELD5", "FIELD6", "FIELD7", "FIELD8", "FIELD9" }; - - /** - * Init. - */ - public void init() throws DBException { - Properties props = getProperties(); - int port = 0; - - String portString = props.getProperty(PORT_PROPERTY); - if (portString != null) { - port = Integer.parseInt(portString); - } - String host = props.getProperty(HOST_PROPERTY); - String tenant = props.getProperty(TENANT_PROPERTY); - String db = props.getProperty(DB_PROPERTY); - String user = props.getProperty(USERNAME_PROPERTY); - String password = props.getProperty(PASSWORD_PROPERTY); - - try { - client = new ObTable.Builder(host, port) // - .setLoginInfo(tenant, user, password, db) // - .build(); - } catch (Exception e) { - throw new DBException(e); - } - - } - - /** - * Read. - */ - @Override - public Status read(String table, String key, Set fields, - HashMap result) { - try { - String[] fs = this.fields; - if (fields != null) { - fs = fields.toArray(new String[] {}); - } - Iterator i$ = client.get(table, key, fs).entrySet().iterator(); - - while (i$.hasNext()) { - Map.Entry entry = (Map.Entry) i$.next(); - result.put(entry.getKey(), new StringByteIterator(entry.getValue().toString())); - } - - return result.isEmpty() ? Status.ERROR : Status.OK; - } catch (Exception e) { - e.printStackTrace(); // NOPMD - } - return null; - } - - /** - * Scan. - */ - @Override - public Status scan(String table, String startkey, int recordcount, Set fields, - Vector> result) { - return null; - } - - /** - * Update. - */ - @Override - public Status update(String table, String key, HashMap values) { - Map vs = StringByteIterator.getStringMap(values); - String[] fields = vs.keySet().toArray(new String[] {}); - String[] fieldValues = vs.values().toArray(new String[] {}); - try { - client.update(table, key, fields, fieldValues); - - return Status.OK; - } catch (Exception e) { - e.printStackTrace(); // NOPMD - } - return Status.ERROR; - } - - /** - * Insert. - */ - @Override - public Status insert(String table, String key, HashMap values) { - Map vs = StringByteIterator.getStringMap(values); - String[] fields = vs.keySet().toArray(new String[] {}); - String[] fieldValues = vs.values().toArray(new String[] {}); - try { - client.insertOrUpdate(table, key, fields, fieldValues); - - return Status.OK; - } catch (Exception e) { - e.printStackTrace(); // NOPMD - } - return Status.ERROR; - } - - /** - * Delete. - */ - @Override - public Status delete(String table, String key) { - try { - return client.delete(table, key) == 0 ? Status.ERROR : Status.OK; - } catch (Exception e) { - e.printStackTrace(); // NOPMD - } - - return Status.ERROR; - } -} diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/YcsbBenchClient.java b/src/main/java/com/alipay/oceanbase/rpc/util/YcsbBenchClient.java deleted file mode 100644 index 1ae27501..00000000 --- a/src/main/java/com/alipay/oceanbase/rpc/util/YcsbBenchClient.java +++ /dev/null @@ -1,208 +0,0 @@ -/*- - * #%L - * OBKV Table Client Framework - * %% - * Copyright (C) 2021 OceanBase - * %% - * OBKV Table Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.rpc.util; - -import com.alipay.oceanbase.rpc.ObTableClient; -import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; -import com.alipay.oceanbase.rpc.stream.QueryResultSet; -import com.alipay.oceanbase.rpc.table.api.TableQuery; -import com.yahoo.ycsb.*; - -import java.util.*; - -public class YcsbBenchClient extends DB { - - public static final String OB_TABLE_CLIENT_FULL_USER_NAME = "ob.table.client.fullUserName"; - public static final String OB_TABLE_CLIENT_PARAM_URL = "ob.table.client.paramURL"; - public static final String OB_TABLE_CLIENT_PASSWORD = "ob.table.client.password"; - public static final String OB_TABLE_CLIENT_SYS_USER_NAME = "ob.table.client.sysUserName"; - public static final String OB_TABLE_CLIENT_SYS_PASSWORD = "ob.table.client.sysPassword"; - public static final String OB_TABLE_CLIENT_SYS_ENC_PASSWORD = "ob.table.client.sysEncPassword"; - - ObTableClient obTableClient; - String[] fields = new String[] { "FIELD0", - "FIELD1", "FIELD2", "FIELD3", "FIELD4", "FIELD5", "FIELD6", "FIELD7", "FIELD8", - "FIELD9" }; - - /** - * Init. - */ - public void init() throws DBException { - Properties props = getProperties(); - - String fullUserName = props.getProperty(OB_TABLE_CLIENT_FULL_USER_NAME); - - if (fullUserName == null) { - throw new DBException("fullUserName can not be null"); - } - - String paramURL = props.getProperty(OB_TABLE_CLIENT_PARAM_URL); - - if (paramURL == null) { - throw new DBException("paramURL can not be null"); - } - - String password = props.getProperty(OB_TABLE_CLIENT_PASSWORD); - - if (password == null) { - password = ""; - } - - String sysUserName = props.getProperty(OB_TABLE_CLIENT_SYS_USER_NAME); - if (password == null) { - throw new DBException("sysUserName can not be null"); - } - - String sysPassword = props.getProperty(OB_TABLE_CLIENT_SYS_PASSWORD); - String encryptedSysPassword = props.getProperty(OB_TABLE_CLIENT_SYS_ENC_PASSWORD); - - try { - final ObTableClient obTableClient = new ObTableClient(); - obTableClient.setFullUserName(fullUserName); - obTableClient.setParamURL(paramURL); - obTableClient.setPassword(password); - obTableClient.setSysUserName(sysUserName); - obTableClient.setSysPassword(sysPassword); - obTableClient.setEncSysPassword(encryptedSysPassword); - obTableClient.init(); - this.obTableClient = obTableClient; - } catch (Exception e) { - throw new DBException(e); - } - - } - - /** - * Read. - */ - @Override - public Status read(String table, String key, Set fields, - HashMap result) { - try { - String[] fs = this.fields; - if (fields != null) { - fs = fields.toArray(new String[] {}); - } - Iterator i$ = obTableClient.get(table, key, fs).entrySet().iterator(); - - while (i$.hasNext()) { - Map.Entry entry = (Map.Entry) i$.next(); - result.put(entry.getKey(), new StringByteIterator(entry.getValue().toString())); - } - - return result.isEmpty() ? Status.ERROR : Status.OK; - } catch (Exception e) { - e.printStackTrace(); // NOPMD - System.err.println("fullUserName:" + obTableClient.getFullUserName() + " url :" // NOPMD - + obTableClient.getParamURL()); // NOPMD - } - return null; - } - - /** - * Scan. - */ - @Override - public Status scan(String table, String startkey, int recordcount, Set fields, - Vector> result) { - - try { - String[] fs = this.fields; - if (fields != null) { - fs = fields.toArray(new String[] {}); - } - TableQuery tableQuery = obTableClient.query(table); - tableQuery.limit(recordcount); - tableQuery.select(fs); - tableQuery.addScanRange(startkey, ObObj.getMax()); - QueryResultSet set = tableQuery.execute(); - - while (set.next()) { - HashMap map = new HashMap(); - Map row = set.getRow(); - for (Map.Entry entry : row.entrySet()) { - map.put(entry.getKey(), new StringByteIterator(entry.getValue().toString())); - } - result.add(map); - } - - return result.isEmpty() ? Status.ERROR : Status.OK; - } catch (Exception e) { - e.printStackTrace(); // NOPMD - System.err.println("fullUserName:" + obTableClient.getFullUserName() + " url :" // NOPMD - + obTableClient.getParamURL()); // NOPMD - } - return null; - } - - /** - * Update. - */ - @Override - public Status update(String table, String key, HashMap values) { - Map vs = StringByteIterator.getStringMap(values); - String[] fields = vs.keySet().toArray(new String[] {}); - String[] fieldValues = vs.values().toArray(new String[] {}); - try { - obTableClient.update(table, key, fields, fieldValues); - - return Status.OK; - } catch (Exception e) { - e.printStackTrace(); // NOPMD - System.err.println("fullUserName:" + obTableClient.getFullUserName() + " url :" // NOPMD - + obTableClient.getParamURL()); // NOPMD - } - return Status.ERROR; - } - - /** - * Insert. - */ - @Override - public Status insert(String table, String key, HashMap values) { - try { - Map vs = StringByteIterator.getStringMap(values); - String[] fields = vs.keySet().toArray(new String[] {}); - String[] fieldValues = vs.values().toArray(new String[] {}); - obTableClient.insertOrUpdate(table, key, fields, fieldValues); - - return Status.OK; - } catch (Exception e) { - e.printStackTrace(); // NOPMD - System.err.println("fullUserName:" + obTableClient.getFullUserName() + " url :" // NOPMD - + obTableClient.getParamURL());// NOPMD - } - return Status.ERROR; - } - - /** - * Delete. - */ - @Override - public Status delete(String table, String key) { - try { - return obTableClient.delete(table, key) == 0 ? Status.ERROR : Status.OK; - } catch (Exception e) { - e.printStackTrace(); // NOPMD - System.err.println("fullUserName:" + obTableClient.getFullUserName() + " url :" // NOPMD - + obTableClient.getParamURL()); // NOPMD - } - - return Status.ERROR; - } -} diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/ZoneUtil.java b/src/main/java/com/alipay/oceanbase/rpc/util/ZoneUtil.java index 7579b66d..eb8d63c4 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/ZoneUtil.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/ZoneUtil.java @@ -20,7 +20,7 @@ public class ZoneUtil { private static String DEFAULT_IDC_VALUE = "dev"; - /** + /* * Get current IDC. */ public static String getCurrentIDC() { diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/hash/MurmurHash.java b/src/main/java/com/alipay/oceanbase/rpc/util/hash/MurmurHash.java index 9e550ce3..6681a937 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/hash/MurmurHash.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/hash/MurmurHash.java @@ -176,6 +176,61 @@ public static long hash64(final byte[] data, int length, long seed) { return h; } + /** + * Generates 64a bit hash from byte array of the given length and seed. + * + * @param data byte array to hash + * @param length length of the array to hash + * @param seed initial seed value + * @return 64 bit hash of the given array + */ + public static long hash64a(final byte[] data, int length, long seed) { + final long m = 0xc6a4a7935bd1e995L; + final int r = 47; + + long h = seed ^ (length * m); + + int endpos = length - (length & 7); + int i = 0; + while (i != endpos) { + long k = (data[i] & 0xFFL) | ((data[i + 1] & 0xFFL) << 8) + | ((data[i + 2] & 0xFFL) << 16) | ((data[i + 3] & 0xFFL) << 24) + | ((data[i + 4] & 0xFFL) << 32) | ((data[i + 5] & 0xFFL) << 40) + | ((data[i + 6] & 0xFFL) << 48) | ((data[i + 7] & 0xFFL) << 56); + + k *= m; + k ^= k >>> r; + k *= m; + h ^= k; + h *= m; + + i += 8; + } + + switch (length & 7) { + case 7: + h ^= (data[i + 6] & 0xFFL) << 48; + case 6: + h ^= (data[i + 5] & 0xFFL) << 40; + case 5: + h ^= (data[i + 4] & 0xFFL) << 32; + case 4: + h ^= (data[i + 3] & 0xFFL) << 24; + case 3: + h ^= (data[i + 2] & 0xFFL) << 16; + case 2: + h ^= (data[i + 1] & 0xFFL) << 8; + case 1: + h ^= (data[i] & 0xFFL); + h *= m; + } + + h ^= h >>> r; + h *= m; + h ^= h >>> r; + return h; + } + /** * Generates 64 bit hash from byte array with default seed value. * diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObHashSortBin.java b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObHashSortBin.java index 296a6a94..e3e66167 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObHashSortBin.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObHashSortBin.java @@ -18,7 +18,7 @@ package com.alipay.oceanbase.rpc.util.hash; public class ObHashSortBin { - /** + /* * Ob hash sort bin. */ public static long obHashSortBin(byte[] s, int len, long n1, long n2) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObHashSortUtf8mb4.java b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObHashSortUtf8mb4.java index 6d3ba782..9dd70d77 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObHashSortUtf8mb4.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObHashSortUtf8mb4.java @@ -30,6 +30,7 @@ public final class ObHashSortUtf8mb4 { public static final int OB_CS_STRNXFRM = 64; // if strnxfrm ictx.start used for sort public static final int OB_CS_UNICODE = 128; // ictx.start a charset ictx.start BMP Unicode public static final int OB_CS_UNICODE_SUPPLEMENT = 16384; // Non-BMP Unicode characters + public static final int OB_HASH_BUFFER_LENGTH = 128; // Utf8mb4 hash buffer length public static ObUnicaseInfoChar[] plane00 = { new ObUnicaseInfoChar(0x0000, 0x0000, 0x0000), @@ -2899,22 +2900,21 @@ public final class ObHashSortUtf8mb4 { | OB_CS_UNICODE | OB_CS_UNICODE_SUPPLEMENT; - /** + /* * A continuation byte in UTF-8 is any byte where the top two bits are 10. - **/ + */ private static boolean isContinuationByte(byte c) { return ((c & 0xC0) == 0x80); } - /** + /* * Whether the buffer has enough bytes remaining. */ private static boolean enoughBytesRemaining(ObMbContext ctx, int n) { return (ctx.end >= ctx.start + n); } - /** - * Please refer to : https://en.wikipedia.org/wiki/UTF-8 + /* * The following table shows how the Unicode code points map to UTF-8 multi-byte byte sequences, * and their equivalent binary values. *

@@ -2926,8 +2926,8 @@ private static boolean enoughBytesRemaining(ObMbContext ctx, int n) { * U+010000-U+10FFFF 000zzzzz yyyyyyyy xxxxxxxx 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx *

* The basic rules are as follows: - * （1） If a byte starts with a 0 bit, it's a single byte value less than 128. - * （2） If it starts with 11, it's the first byte of a multi-byte sequence + * (1) If a byte starts with a 0 bit, it's a single byte value less than 128. + * (2) If it starts with 11, it's the first byte of a multi-byte sequence * and the number of 1 bits at the start indicates how many bytes there are in total, * eg., 110xxxxx has two bytes, 1110xxxx has three and 11110xxx has four. * (3) If it starts with 10, it's a continuation byte. @@ -3021,37 +3021,66 @@ private static void obHashAdd(ObMbContext ctx, int ch) { ctx.n2 += 3; } - /** + /* * Ob hash sort utf8 mb4. */ - public static long obHashSortUtf8Mb4(byte[] s, int len, long n1, long n2) { + public static long obHashSortUtf8Mb4(byte[] s, int len, long n1, long n2, boolean hash_algo) { ObMbContext ctx = new ObMbContext(); ctx.buf = s; ctx.n1 = n1; ctx.n2 = n2; ctx.start = 0; ctx.end = len; + ctx.hashBuf = new byte[OB_HASH_BUFFER_LENGTH]; + ctx.len = 0; ObUnicaseInfo uniPlane = obUnicaseDefault; while (ctx.end > ctx.start && ctx.buf[ctx.end - 1] == ' ') { ctx.end--; } - int res; - while ((res = obMbWcUtf8Mb4(ctx)) > 0) { - obTosortUnicode(uniPlane, ctx, state); - obHashAdd(ctx, (int) (ctx.pwc & 0xFF)); - obHashAdd(ctx, (int) (ctx.pwc >> 8) & 0xFF); - if (ctx.pwc > 0xFFFF) { - obHashAdd(ctx, (int) (ctx.pwc >> 16) & 0xFF); + if (hash_algo) { + int res; + while ((res = obMbWcUtf8Mb4(ctx)) > 0) { + obTosortUnicode(uniPlane, ctx, state); + if (ctx.len > OB_HASH_BUFFER_LENGTH - 2 + || (OB_HASH_BUFFER_LENGTH - 2 == ctx.len && ctx.pwc > 0xFFFF)) { + ctx.n1 = MurmurHash.hash64a(ctx.hashBuf, ctx.len, ctx.n1); + ctx.len = 0; + } + + ctx.hashBuf[ctx.len] = (byte) (ctx.pwc & 0xFF); + ctx.len += 1; + ctx.hashBuf[ctx.len] = (byte) ((ctx.pwc >> 8) & 0xFF); + ctx.len += 1; + if (ctx.pwc > 0xFFFF) { + ctx.hashBuf[ctx.len] = (byte) ((ctx.pwc >> 16) & 0xFF); + ctx.len += 1; + } + + ctx.start += res; + } + if (ctx.len > 0) { + ctx.n1 = MurmurHash.hash64a(ctx.hashBuf, ctx.len, ctx.n1); + ctx.len = 0; + } + } else { + int res; + while ((res = obMbWcUtf8Mb4(ctx)) > 0) { + obTosortUnicode(uniPlane, ctx, state); + obHashAdd(ctx, (int) (ctx.pwc & 0xFF)); + obHashAdd(ctx, (int) (ctx.pwc >> 8) & 0xFF); + if (ctx.pwc > 0xFFFF) { + obHashAdd(ctx, (int) (ctx.pwc >> 16) & 0xFF); + } + ctx.start += res; } - ctx.start += res; } return ctx.n1; } - /** - * Ob hash sort mb bin. + /* + * Ob hash sort mb bin. Used by observer2.x */ public static long obHashSortMbBin(byte[] s, int len, long n1, long n2) { // skip_trailing_space diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObMbContext.java b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObMbContext.java index 1a3d0b37..847e992f 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObMbContext.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObMbContext.java @@ -24,4 +24,7 @@ public class ObMbContext { int end; long n1; long n2; + // for 3.x + byte[] hashBuf; + int len; } diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObUnicaseInfo.java b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObUnicaseInfo.java index a3f3332b..650d484a 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObUnicaseInfo.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObUnicaseInfo.java @@ -18,7 +18,7 @@ package com.alipay.oceanbase.rpc.util.hash; public class ObUnicaseInfo { - /** + /* * Ob unicase info. */ public ObUnicaseInfo(int maxchar, ObUnicaseInfoChar[][] page) { diff --git a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObUnicaseInfoChar.java b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObUnicaseInfoChar.java index 83969c68..a8b0f4bd 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObUnicaseInfoChar.java +++ b/src/main/java/com/alipay/oceanbase/rpc/util/hash/ObUnicaseInfoChar.java @@ -22,7 +22,7 @@ public class ObUnicaseInfoChar { int tolower; int sort; - /** + /* * Ob unicase info char. */ public ObUnicaseInfoChar(int toupper, int tolower, int sort) { diff --git a/src/main/resources/oceanbase-table-client/log-codes.properties b/src/main/resources/oceanbase-table-client/log-codes.properties index 7b91ac4c..63754388 100644 --- a/src/main/resources/oceanbase-table-client/log-codes.properties +++ b/src/main/resources/oceanbase-table-client/log-codes.properties @@ -3,7 +3,7 @@ 01-00003=Exception caught! 01-00004=close obTable {} error 01-00005=fail to decode proxyro password, errMsg={} -01-00006=fail to find com.mysql.jdbc.Driver, key={}, errMsg={} +01-00006=fail to find jdbc driver, key={}, errMsg={} 01-00007=fail to refresh table entry from remote url={}, key={} 01-00008=table entry is invalid, addr = {}, key = {}, entry = {} 01-00009=fail to get table entry from remote, key={} diff --git a/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml b/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml index 2def4c82..403587ff 100644 --- a/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml +++ b/src/main/resources/oceanbase-table-client/log/log4j/log-conf.xml @@ -5,7 +5,7 @@ - + @@ -15,8 +15,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/oceanbase-table-client/log/log4j2/log-conf.xml b/src/main/resources/oceanbase-table-client/log/log4j2/log-conf.xml index 1ddb97ab..5aece116 100644 --- a/src/main/resources/oceanbase-table-client/log/log4j2/log-conf.xml +++ b/src/main/resources/oceanbase-table-client/log/log4j2/log-conf.xml @@ -21,12 +21,63 @@ + + + + + %d-%m%n + + + + + + + + + + + %d-%m%n + + + + + + + + + + + %d-%m%n + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/resources/oceanbase-table-client/log/logback/log-conf.xml b/src/main/resources/oceanbase-table-client/log/logback/log-conf.xml index 90102a8a..8aae2c41 100644 --- a/src/main/resources/oceanbase-table-client/log/logback/log-conf.xml +++ b/src/main/resources/oceanbase-table-client/log/logback/log-conf.xml @@ -2,21 +2,120 @@ - ${logging.path.com.alipay.oceanbase-table-client}/oceanbase-table-client/oceanbase-table-client.log + ${logging.path:-.}/oceanbase-table-client/oceanbase-table-client.log + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] - %msg%n + + + + + ${logging.path:-.}/oceanbase-table-client/oceanbase-table-client.%d{yyyy-MM-dd}.%i.log + false + 30 + 100MB + + + + + ${logging.path:-.}/oceanbase-table-client/oceanbase-table-client-boot.log + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{50} - [%C{0}#%L] - %msg%n + + + + + ${logging.path:-.}/oceanbase-table-client/oceanbase-table-client-boot.%d{yyyy-MM-dd}.%i.log + false + 30 + 100MB + + + + + ${logging.path:-.}/oceanbase-table-client/oceanbase-table-client-monitor.log - %d{yyyy-MM-dd HH:mm:ss}-%msg%n + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{50} - %msg%n - + - ${logging.path.oceanbase-table-client}/oceanbase-table-client/oceanbase-table-client.%d{yyyy-MM-dd}.log - + ${logging.path:-.}/oceanbase-table-client/oceanbase-table-client-monitor.%d{yyyy-MM-dd}.%i.log + false + 30 + 500MB + 20GB - + + ${logging.path:-.}/oceanbase-table-client/oceanbase-table-client-runtime.log + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{50} - %msg%n + + + + + ${logging.path:-.}/oceanbase-table-client/oceanbase-table-client-runtime.%d{yyyy-MM-dd}.%i.log + false + 30 + 100MB + + + + + + 0 + false + 10240 + false + + + + + 0 + false + 1024 + false + + + + 0 + false + 1024 + false + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/test/java/ci.sql b/src/test/java/ci.sql deleted file mode 100644 index 0104902f..00000000 --- a/src/test/java/ci.sql +++ /dev/null @@ -1,128 +0,0 @@ -CREATE TABLE IF NOT EXISTS `test_varchar_table` ( - `c1` varchar(20) NOT NULL, - `c2` varchar(20) DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_blob_table` ( - `c1` varchar(20) NOT NULL, - `c2` blob DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_longblob_table` ( - `c1` varchar(20) NOT NULL, - `c2` longblob DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_varchar_table_for_exception` ( - `c1` varchar(20) NOT NULL, - `c2` varchar(20) NOT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_tinyint_table` ( - `c1` varchar(20) NOT NULL, - `c2` tinyint(4) DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_smallint_table` ( - `c1` varchar(20) NOT NULL, - `c2` smallint(8) DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_int_table` ( - `c1` int(12) NOT NULL, - `c2` int(12) DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_bigint_table` ( - `c1` bigint(20) NOT NULL, - `c2` bigint(20) DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_double_table` ( - `c1` varchar(20) NOT NULL, - `c2` double DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_float_table` ( - `c1` varchar(20) NOT NULL, - `c2` float DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_varbinary_table` ( - `c1` varchar(20) NOT NULL, - `c2` varbinary(20) DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_timestamp_table` ( - `c1` varchar(20) NOT NULL, - `c2` timestamp DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `test_datetime_table` ( - `c1` varchar(20) NOT NULL, - `c2` datetime DEFAULT NULL, - PRIMARY KEY (`c1`) - ); - -CREATE TABLE IF NOT EXISTS `testHash`( - `K` bigint, - `Q` varbinary(256), - `T` bigint, - `V` varbinary(1024), - PRIMARY KEY(`K`, `Q`, `T`) -) partition by hash(`K`) partitions 16; - -CREATE TABLE IF NOT EXISTS `testPartition` ( - `K` varbinary(1024), - `Q` varbinary(256), - `T` bigint, - `V` varbinary(1024), - K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4)), - PRIMARY KEY(`K`, `Q`, `T`) -) partition by key(K_PREFIX) partitions 15; - -CREATE TABLE IF NOT EXISTS `test_increment` ( - `c1` varchar(255), - `c2` int, - `c3` int, - PRIMARY KEY(`c1`) -); - -CREATE TABLE IF NOT EXISTS `test_append`( - `c1` varchar(255), - `c2` varbinary(1024), - `c3` varchar(255), - PRIMARY KEY(`c1`) -); - -CREATE TABLE IF NOT EXISTS `testRange` ( - `K` varbinary(1024), - `Q` varbinary(256), - `T` bigint, - `V` varbinary(102400), - PRIMARY KEY(`K`, `Q`, `T`) -) partition by range columns (`K`) ( - PARTITION p0 VALUES LESS THAN ('a'), - PARTITION p1 VALUES LESS THAN ('w'), - PARTITION p2 VALUES LESS THAN MAXVALUE -); - -CREATE TABLE IF NOT EXISTS `test_hbase$fn` ( - `K` varbinary(1024) NOT NULL, - `Q` varbinary(256) NOT NULL, - `T` bigint(20) NOT NULL, - `V` varbinary(1024) DEFAULT NULL, - PRIMARY KEY (`K`, `Q`, `T`) -); \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java index c74d2926..74dbefb3 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObAtomicBatchOperationTest.java @@ -18,6 +18,13 @@ package com.alipay.oceanbase.rpc; import com.alipay.oceanbase.rpc.exception.ObTableDuplicateKeyException; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.mutation.BatchOperation; +import com.alipay.oceanbase.rpc.mutation.Delete; +import com.alipay.oceanbase.rpc.mutation.Insert; +import com.alipay.oceanbase.rpc.mutation.Put; +import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; import com.alipay.oceanbase.rpc.table.api.TableBatchOps; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; import org.junit.After; @@ -25,9 +32,13 @@ import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.row; + public class ObAtomicBatchOperationTest { private static final int dataSetSize = 4; private static final String successKey = "abc-5"; @@ -62,6 +73,10 @@ public void teardown() throws Exception { obTableClient.delete("test_varchar_table", key); } obTableClient.delete("test_varchar_table", successKey); + for (int i = 1; i <= 9; i++) { + String key = "abcd-" + i; + obTableClient.delete("test_varchar_table", key); + } obTableClient.close(); } @@ -76,9 +91,9 @@ public void testAtomic() { batchOps.insert("abc-3", new String[] { "c2" }, new String[] { "bar-3" }); batchOps.insert(successKey, new String[] { "c2" }, new String[] { "bar-5" }); List results = batchOps.execute(); - Assert.assertTrue(results.get(0) instanceof ObTableDuplicateKeyException); + Assert.assertTrue(results.get(0) instanceof ObTableException); Assert.assertEquals(((Map) results.get(1)).get("c2"), "xyz-2"); - Assert.assertTrue(results.get(2) instanceof ObTableDuplicateKeyException); + Assert.assertTrue(results.get(2) instanceof ObTableException); Assert.assertEquals(results.get(3), 1L); } catch (Exception ex) { Assert.fail("hit exception:" + ex); @@ -96,8 +111,273 @@ public void testAtomic() { // no support atomic batch // Assert.fail("expect duplicate key exception."); } catch (Exception ex) { - Assert.assertTrue(ex instanceof ObTableDuplicateKeyException); + Assert.assertTrue(ex instanceof ObTableException); } } + @Test + public void testBatchOperation() { + TableBatchOps batchOps = obTableClient.batch("test_varchar_table"); + try { + // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: false + { + batchOps.clear(); + batchOps.insert("abc-1", new String[] { "c1", "c2" }, new String[] { "bar-1", + "bar-2" }); + batchOps.get("abc-2", new String[] { "c2" }); + batchOps.insert("abc-3", new String[] { "c2" }, new String[] { "bar-3" }); + + Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + // 测试 isReadOnly: true, isSameType: true, isSamePropertiesNames: false + { + batchOps.clear(); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c2", "c4" }); + batchOps.get("abc-4", new String[] { "c1", "c2" }); + + Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + + batchOps.clear(); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c2", "c4" }); + batchOps.get("abc-4", new String[] { "c1", "c2", "c3" }); + + Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + // 测试 isReadOnly: true, isSameType: true, isSamePropertiesNames: true + { + batchOps.clear(); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-4", new String[] { "c1", "c2", "c3" }); + + Assert.assertTrue(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: true + { + batchOps.clear(); + batchOps.get("abc-2", new String[] { "c1a", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c2", "c3", "C1A" }); + batchOps.get("abc-4", new String[] { "c1A", "c2", "c3" }); + batchOps.insert("abc-4", new String[] { "c3", "C2", "c1a" }, new String[] { + "bar-3", "bar-3", "bar-3" }); + batchOps.insert("abc-4", new String[] { "c1A", "c2", "C3" }, new String[] { + "bar-2", "bar-2", "bar-2" }); + + Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertTrue(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + // 测试 isReadOnly: false, isSameType: false, isSamePropertiesNames: false + { + batchOps.clear(); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-4", new String[] { "c1", "c2" }); + batchOps.insert("abc-4", new String[] { "c1", "c2", "c3" }, new String[] { "bar-3", + "bar-3", "bar-3" }); + batchOps.insert("abc-4", new String[] { "c1", "c2", "c3" }, new String[] { "bar-2", + "bar-2", "bar-2" }); + + Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + + batchOps.clear(); + batchOps.get("abc-2", new String[] { "c1", "c2", "c3" }); + batchOps.get("abc-3", new String[] { "c1", "c4", "c3" }); + batchOps.insert("abc-4", new String[] { "c1", "c2", "c3" }, new String[] { "bar-3", + "bar-3", "bar-3" }); + batchOps.insert("abc-4", new String[] { "c2", "c3", "c1" }, new String[] { "bar-2", + "bar-2", "bar-2" }); + + Assert.assertFalse(batchOps.getObTableBatchOperation().isReadOnly()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSameType()); + Assert.assertFalse(batchOps.getObTableBatchOperation().isSamePropertiesNames()); + } + + } catch (Exception ex) { + Assert.fail("hit exception:" + ex); + } + + } + + @Test + public void testReturnOneRes() { + TableBatchOps batchOps = obTableClient.batch("test_varchar_table"); + // no atomic ReturnOneRes batch operation + try { + batchOps.clear(); + batchOps.setAtomicOperation(false); + batchOps.setReturnOneResult(true); + batchOps.insert("abcd-7", new String[] { "c2" }, new String[] { "returnOne-7" }); + batchOps.insert("abcd-8", new String[] { "c2" }, new String[] { "returnOne-8" }); + batchOps.insert("abcd-9", new String[] { "c2" }, new String[] { "returnOne-9" }); + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 1); + Assert.assertEquals(results.get(0), 3L); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof ObTableException); + } + + // atomic ReturnOneRes batch operation + try { + batchOps.clear(); + batchOps.setAtomicOperation(true); + batchOps.setReturnOneResult(true); + batchOps.insert("abcd-4", new String[] { "c2" }, new String[] { "returnOne-4" }); + batchOps.insert("abcd-5", new String[] { "c2" }, new String[] { "returnOne-5" }); + batchOps.insert("abcd-6", new String[] { "c2" }, new String[] { "returnOne-6" }); + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 1); + Assert.assertEquals(results.get(0), 3L); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof ObTableException); + } + + // batch delete operation + try { + // del + batchOps.clear(); + batchOps.setReturnOneResult(true); + batchOps.setAtomicOperation(false); + for (int i = 0; i <= 9; i += 2) { + String key = "abc-" + i; + batchOps.delete(key); + } + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 1); + Assert.assertEquals(results.get(0), 2L); + // get + batchOps.clear(); + batchOps.setAtomicOperation(true); + for (int i = 0; i <= 9; i++) { + String key = "abc-" + i; + batchOps.get(key, new String[] { "c2" }); + } + batchOps.setReturnOneResult(false); + List results_get = batchOps.execute(); + + Assert.assertTrue(((Map) results_get.get(0)).isEmpty()); + Assert.assertFalse(((Map) results_get.get(1)).isEmpty()); + Assert.assertTrue(((Map) results_get.get(2)).isEmpty()); + Assert.assertFalse(((Map) results_get.get(3)).isEmpty()); + Assert.assertTrue(((Map) results_get.get(4)).isEmpty()); + Assert.assertTrue(((Map) results_get.get(5)).isEmpty()); + + } catch (Exception ex) { + Assert.assertTrue(ex instanceof ObTableException); + } + + // default: no ReturnOneRes batch operation + try { + batchOps.clear(); + batchOps.setAtomicOperation(false); + batchOps.setReturnOneResult(false); + batchOps.insert("abcd-1", new String[] { "c2" }, new String[] { "returnOne-1" }); + batchOps.insert("abcd-2", new String[] { "c2" }, new String[] { "returnOne-2" }); + batchOps.insert("abcd-3", new String[] { "c2" }, new String[] { "returnOne-3" }); + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 3); + Assert.assertEquals(results.get(0), 1L); + Assert.assertEquals(results.get(1), 1L); + Assert.assertEquals(results.get(2), 1L); + } catch (Exception ex) { + Assert.fail("hit exception:" + ex); + } + } + + @Test + public void testReturnOneResPartition() throws Exception { + BatchOperation batchOperation = obTableClient.batchOperation("test_mutation"); + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 200L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 2000L, "c2_val", "c3_val", 100L }, + { 100001L, "c2_val", "c3_val", 100L }, { 10000002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(batchOperationResult.size(), 1); + Assert.assertEquals(batchOperationResult.get(0).getAffectedRows(), 6); + } catch (Exception ex) { + Assert.fail("hit exception:" + ex); + } finally { + for (int j = 0; j < rowCnt; j++) { + Object[] curRow = values[j]; + obTableClient.delete("test_mutation", new Object[] { curRow[0], curRow[1] }); + + } + } + } + + @Test + public void testBatchGet() throws Exception { + try { + { + // insert + TableBatchOps batchOps = obTableClient.batch("test_varchar_table"); + batchOps.clear(); + batchOps.insert("batch_get-1", new String[] { "c2" }, new String[] { "bar-1" }); + batchOps.insert("batch_get-2", new String[] { "c2" }, new String[] { "bar-2" }); + batchOps.insert("batch_get-3", new String[] { "c2" }, new String[] { "bar-3" }); + batchOps.insert("batch_get-4", new String[] { "c2" }, new String[] { "bar-4" }); + batchOps.setReturnOneResult(true); + List results = batchOps.execute(); + Assert.assertEquals(results.get(0), 4L); + } + { + // get + TableBatchOps batchOps = obTableClient.batch("test_varchar_table"); + batchOps.clear(); + batchOps.get("batch_get-1", new String[] { "c2" }); + batchOps.get("batch_get-2", new String[] { "c2" }); + batchOps.get("batch_get-3", new String[] { "c2" }); + batchOps.get("batch_get-4", new String[] { "c2" }); + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 4L); + Assert.assertEquals(((Map) results.get(0)).get("c2"), "bar-1"); + Assert.assertEquals(((Map) results.get(1)).get("c2"), "bar-2"); + Assert.assertEquals(((Map) results.get(2)).get("c2"), "bar-3"); + Assert.assertEquals(((Map) results.get(3)).get("c2"), "bar-4"); + } + { + // get + TableBatchOps batchOps = obTableClient.batch("test_varchar_table"); + batchOps.clear(); + batchOps.get("batch_get-1", new String[] { "c2" }); + batchOps.get("batch_get-2", new String[] { "c2" }); + batchOps.get("batch_get-1", new String[] { "c2" }); + batchOps.get("batch_get-2", new String[] { "c2" }); + List results = batchOps.execute(); + Assert.assertEquals(results.size(), 4L); + Assert.assertEquals(((Map) results.get(0)).get("c2"), "bar-1"); + Assert.assertEquals(((Map) results.get(1)).get("c2"), "bar-2"); + Assert.assertEquals(((Map) results.get(2)).get("c2"), "bar-1"); + Assert.assertEquals(((Map) results.get(3)).get("c2"), "bar-2"); + } + } catch (Exception ex) { + Assert.fail("hit exception:" + ex); + } finally { + for (int i = 1; i <= 4; i++) { + String key = "batch_get-" + i; + obTableClient.delete("test_varchar_table", key); + } + } + } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableAggregationTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableAggregationTest.java new file mode 100644 index 00000000..0486c18b --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableAggregationTest.java @@ -0,0 +1,633 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.filter.*; +import com.alipay.oceanbase.rpc.property.Property; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation.ObTableAggregation; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.aggregation.ObTableAggregationResult; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.text.SimpleDateFormat; +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +public class ObTableAggregationTest { + private ObTableClient client; + + @Before + public void setup() throws Exception { + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.setMetadataRefreshInterval(100); + obTableClient.addProperty(Property.RPC_CONNECT_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.RPC_LOGIN_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.SERVER_CONNECTION_POOL_SIZE.getKey(), "1"); + obTableClient.addProperty(Property.RPC_EXECUTE_TIMEOUT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_MAX_WAIT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_EXECUTOR.getKey(), "32"); + obTableClient.addProperty(Property.RPC_OPERATION_TIMEOUT.getKey(), "3000"); + obTableClient.addProperty(Property.SERVER_ENABLE_REROUTING.getKey(), "False"); + obTableClient.init(); + + this.client = obTableClient; + } + + @Test + // Test aggregation + public void testAggregation() throws Exception { + + /* + * CREATE TABLE test_aggregation ( + * `c1` varchar(255), + * `c2` int NOT NULL, + * `c3` bigint NOT NULL, + * `c4` float NOT NULL, + * `c5` double NOT NULL, + * `c6` tinyint NULL, + * `c7` datetime, + * PRIMARY KEY(`c1`) + * ); + * */ + + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_aggregation", new String[] { "c1" }); + + SimpleDateFormat sdf = new SimpleDateFormat(" yyyy-MM-dd HH:mm:ss "); + Date date1 = sdf.parse(" 2001-07-10 19:20:00 "); + Date date2 = sdf.parse(" 2002-07-10 19:20:00 "); + Date date3 = sdf.parse(" 2003-07-10 19:20:00 "); + + try { + client.insert("test_aggregation", "first_row", new String[] { "c2", "c3", "c4", "c5", + "c6", "c7" }, new Object[] { 1, 1L, 1.0f, 1.0, (byte) 1, date1 }); + client.insert("test_aggregation", "second_row", new String[] { "c2", "c3", "c4", "c5", + "c6", "c7" }, new Object[] { 2, 2L, 2.0f, 2.0, (byte) 2, date2 }); + client.insert("test_aggregation", "third_row", new String[] { "c2", "c3", "c4", "c5", + "c6", "c7" }, new Object[] { 3, 3L, 3.0f, 3.0, (byte) 3, date3 }); + ObTableAggregation obtableAggregation = client.aggregate("test_aggregation"); + + // test int + obtableAggregation.max("c2"); + obtableAggregation.min("c2"); + obtableAggregation.count(); + obtableAggregation.sum("c2"); + obtableAggregation.avg("c2"); + + // test bigint + obtableAggregation.max("c3"); + obtableAggregation.min("c3"); + obtableAggregation.count(); + obtableAggregation.sum("c3"); + obtableAggregation.avg("c3"); + + // test float + obtableAggregation.max("c4"); + obtableAggregation.min("c4"); + obtableAggregation.count(); + obtableAggregation.sum("c4"); + obtableAggregation.avg("c4"); + + // test double + obtableAggregation.max("c5"); + obtableAggregation.min("c5"); + obtableAggregation.count(); + obtableAggregation.sum("c5"); + obtableAggregation.avg("c5"); + + // test tinyint + obtableAggregation.max("c6"); + obtableAggregation.min("c6"); + obtableAggregation.count(); + obtableAggregation.sum("c6"); + obtableAggregation.avg("c6"); + + // test date + obtableAggregation.max("c7"); + obtableAggregation.min("c7"); + + // execute + ObTableAggregationResult obtableAggregationResult = obtableAggregation.execute(); + + // test int + Assert.assertEquals(3, obtableAggregationResult.get("max(c2)")); + Assert.assertEquals(1, obtableAggregationResult.get("min(c2)")); + Assert.assertEquals(3L, obtableAggregationResult.get("count(*)")); + Assert.assertEquals(6L, obtableAggregationResult.get("sum(c2)")); + Assert.assertEquals(2.0, obtableAggregationResult.get("avg(c2)")); + + // test bigint + Assert.assertEquals(3L, obtableAggregationResult.get("max(c3)")); + Assert.assertEquals(1L, obtableAggregationResult.get("min(c3)")); + Assert.assertEquals(3L, obtableAggregationResult.get("count(*)")); + Assert.assertEquals(6L, obtableAggregationResult.get("sum(c3)")); + Assert.assertEquals(2.0, obtableAggregationResult.get("avg(c3)")); + + // test float + Assert.assertEquals(3.0f, obtableAggregationResult.get("max(c4)")); + Assert.assertEquals(1.0f, obtableAggregationResult.get("min(c4)")); + Assert.assertEquals(3L, obtableAggregationResult.get("count(*)")); + Assert.assertEquals(6.0, obtableAggregationResult.get("sum(c4)")); + Assert.assertEquals(2.0, obtableAggregationResult.get("avg(c4)")); + + // test double + Assert.assertEquals(3.0, obtableAggregationResult.get("max(c5)")); + Assert.assertEquals(1.0, obtableAggregationResult.get("min(c5)")); + Assert.assertEquals(3L, obtableAggregationResult.get("count(*)")); + Assert.assertEquals(6.0, obtableAggregationResult.get("sum(c5)")); + Assert.assertEquals(2.0, obtableAggregationResult.get("avg(c5)")); + + // test tinyint + Assert.assertEquals((byte) 3, obtableAggregationResult.get("max(c6)")); + Assert.assertEquals((byte) 1, obtableAggregationResult.get("min(c6)")); + Assert.assertEquals(3L, obtableAggregationResult.get("count(*)")); + Assert.assertEquals(6L, obtableAggregationResult.get("sum(c6)")); + Assert.assertEquals(2.0, obtableAggregationResult.get("avg(c6)")); + + //test date + Assert.assertEquals(date3, obtableAggregationResult.get("max(c7)")); + Assert.assertEquals(date1, obtableAggregationResult.get("min(c7)")); + + } finally { + client.delete("test_aggregation", "first_row"); + client.delete("test_aggregation", "second_row"); + client.delete("test_aggregation", "third_row"); + } + } + + @Test + // Test aggregation of multiple aggregation + public void testPartitionAggregation() throws Exception { + + /* + * CREATE TABLE `test_partition_aggregation` ( + *`c1` bigint NOT NULL, + *`c2` bigint DEFAULT NULL, + *PRIMARY KEY (`c1`))partition by range(`c1`)(partition p0 values less than(200), partition p1 values less than(500), partition p2 values less than(900)); + */ + + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_partition_aggregation", new String[] { "c1" }); + + // test one partition + try { + client.insert("test_partition_aggregation", new Object[] { 50L }, + new String[] { "c2" }, new Object[] { 50L }); + client.insert("test_partition_aggregation", new Object[] { 150L }, + new String[] { "c2" }, new Object[] { 150L }); + client.insert("test_partition_aggregation", new Object[] { 300L }, + new String[] { "c2" }, new Object[] { 300L }); + + ObTableAggregation obtableAggregation = client.aggregate("test_partition_aggregation"); + + obtableAggregation.addScanRange(new Object[] { 0L }, new Object[] { 150L }); + + // test + obtableAggregation.max("c2"); + obtableAggregation.min("c2"); + obtableAggregation.count(); + obtableAggregation.sum("c2"); + obtableAggregation.avg("c2"); + + ObTableException obTableException = null; + + try { + // execute + ObTableAggregationResult obtableAggregationResult = obtableAggregation.execute(); + + // test + Assert.assertEquals(150L, obtableAggregationResult.get("max(c2)")); + Assert.assertEquals(50L, obtableAggregationResult.get("min(c2)")); + Assert.assertEquals(2L, obtableAggregationResult.get("count(*)")); + Assert.assertEquals(200L, obtableAggregationResult.get("sum(c2)")); + Assert.assertEquals(100.0, obtableAggregationResult.get("avg(c2)")); + + } catch (ObTableException e) { + System.out.println(e.getMessage()); + System.out.println(e.getErrorCode()); + fail(); + } + + } finally { + client.delete("test_partition_aggregation", 50L); + client.delete("test_partition_aggregation", 150L); + client.delete("test_partition_aggregation", 300L); + } + + // test multiple partitions + try { + client.insert("test_partition_aggregation", new Object[] { 50L }, + new String[] { "c2" }, new Object[] { 50L }); + client.insert("test_partition_aggregation", new Object[] { 150L }, + new String[] { "c2" }, new Object[] { 150L }); + client.insert("test_partition_aggregation", new Object[] { 300L }, + new String[] { "c2" }, new Object[] { 300L }); + + ObTableAggregation obtableAggregation = client.aggregate("test_partition_aggregation"); + + obtableAggregation.addScanRange(new Object[] { 0L }, new Object[] { 300L }); + + // test + obtableAggregation.max("c2"); + obtableAggregation.min("c2"); + obtableAggregation.count(); + obtableAggregation.sum("c2"); + obtableAggregation.avg("c2"); + + ObTableException obTableException = null; + + try { + // execute + ObTableAggregationResult obtableAggregationResult = obtableAggregation.execute(); + + // test + Assert.assertEquals(150L, obtableAggregationResult.get("max(c2)")); + Assert.assertEquals(50L, obtableAggregationResult.get("min(c2)")); + Assert.assertEquals(2L, obtableAggregationResult.get("count(*)")); + Assert.assertEquals(200L, obtableAggregationResult.get("sum(c2)")); + Assert.assertEquals(100.0, obtableAggregationResult.get("avg(c2)")); + + } catch (ObTableException e) { + obTableException = e; + assertNotNull(obTableException); + assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, e.getErrorCode()); + } + + } finally { + client.delete("test_partition_aggregation", 50L); + client.delete("test_partition_aggregation", 150L); + client.delete("test_partition_aggregation", 300L); + } + } + + @Test + // Test aggregation with filter + public void testAggregationWithFilter() throws Exception { + + /* + * CREATE TABLE `test_partition_aggregation` ( + *`c1` bigint NOT NULL, + *`c2` bigint DEFAULT NULL, + *PRIMARY KEY (`c1`))partition by range(`c1`)(partition p0 values less than(200), partition p1 values less than(500), partition p2 values less than(900)); + */ + + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_partition_aggregation", new String[] { "c1" }); + + // test without filter + try { + client.insert("test_partition_aggregation", new Object[] { 50L }, + new String[] { "c2" }, new Object[] { 50L }); + client.insert("test_partition_aggregation", new Object[] { 100L }, + new String[] { "c2" }, new Object[] { 100L }); + client.insert("test_partition_aggregation", new Object[] { 120L }, + new String[] { "c2" }, new Object[] { 300L }); + client.insert("test_partition_aggregation", new Object[] { 130L }, + new String[] { "c2" }, new Object[] { 300L }); + + // without filter + ObTableAggregation obtableAggregationWithoutFilter = client + .aggregate("test_partition_aggregation"); + + obtableAggregationWithoutFilter + .addScanRange(new Object[] { 0L }, new Object[] { 150L }); + + // test + obtableAggregationWithoutFilter.max("c2"); + obtableAggregationWithoutFilter.min("c2"); + obtableAggregationWithoutFilter.count(); + obtableAggregationWithoutFilter.sum("c2"); + obtableAggregationWithoutFilter.avg("c2"); + + // execute + ObTableAggregationResult obtableAggregationResultWithoutFilter = obtableAggregationWithoutFilter + .execute(); + + // test without filter + Assert.assertEquals(300L, obtableAggregationResultWithoutFilter.get("max(c2)")); + Assert.assertEquals(50L, obtableAggregationResultWithoutFilter.get("min(c2)")); + Assert.assertEquals(4L, obtableAggregationResultWithoutFilter.get("count(*)")); + Assert.assertEquals(750L, obtableAggregationResultWithoutFilter.get("sum(c2)")); + Assert.assertEquals(187.5, obtableAggregationResultWithoutFilter.get("avg(c2)")); + + // with filter + ObTableAggregation obtableAggregationWithFilter = client + .aggregate("test_partition_aggregation"); + + obtableAggregationWithFilter.addScanRange(new Object[] { 0L }, new Object[] { 150L }); + + // test + obtableAggregationWithFilter.max("c2"); + obtableAggregationWithFilter.min("c2"); + obtableAggregationWithFilter.count(); + obtableAggregationWithFilter.sum("c2"); + obtableAggregationWithFilter.avg("c2"); + + // filter + ObTableValueFilter filter = new ObTableValueFilter(ObCompareOp.GT, "c1", 90L); + + // add filter + obtableAggregationWithFilter.setFilter(filter); + + // execute + ObTableAggregationResult obtableAggregationResultWithFilter = obtableAggregationWithFilter + .execute(); + + // test with filter + double delta = 1e-6; + Assert.assertEquals(300L, obtableAggregationResultWithFilter.get("max(c2)")); + Assert.assertEquals(100L, obtableAggregationResultWithFilter.get("min(c2)")); + Assert.assertEquals(3L, obtableAggregationResultWithFilter.get("count(*)")); + Assert.assertEquals(700L, obtableAggregationResultWithFilter.get("sum(c2)")); + Assert.assertEquals(233.33333333, + (double) obtableAggregationResultWithFilter.get("avg(c2)"), delta); + + // with filter + ObTableAggregation obtableAggregationWithFilterAndLimit = client + .aggregate("test_partition_aggregation"); + + obtableAggregationWithFilterAndLimit.addScanRange(new Object[] { 0L }, + new Object[] { 150L }); + + // test + obtableAggregationWithFilterAndLimit.max("c2"); + obtableAggregationWithFilterAndLimit.min("c2"); + obtableAggregationWithFilterAndLimit.count(); + obtableAggregationWithFilterAndLimit.sum("c2"); + obtableAggregationWithFilterAndLimit.avg("c2"); + + // add filter + obtableAggregationWithFilterAndLimit.setFilter(filter); + + // add limit + obtableAggregationWithFilterAndLimit.limit(0, 2); + + // execute + ObTableAggregationResult obtableAggregationResultWithFilterAndLimit = obtableAggregationWithFilterAndLimit + .execute(); + + // test with filter and limit + Assert.assertEquals(300L, obtableAggregationResultWithFilterAndLimit.get("max(c2)")); + Assert.assertEquals(100L, obtableAggregationResultWithFilterAndLimit.get("min(c2)")); + Assert.assertEquals(2L, obtableAggregationResultWithFilterAndLimit.get("count(*)")); + Assert.assertEquals(400L, obtableAggregationResultWithFilterAndLimit.get("sum(c2)")); + Assert.assertEquals(200.0, obtableAggregationResultWithFilterAndLimit.get("avg(c2)")); + + } finally { + client.delete("test_partition_aggregation", 50L); + client.delete("test_partition_aggregation", 100L); + client.delete("test_partition_aggregation", 120L); + client.delete("test_partition_aggregation", 130L); + } + } + + @Test + // Test aggregation with empty table + public void testAggregationWithEmptyRow() throws Exception { + /* + * CREATE TABLE `test_partition_aggregation` ( + *`c1` bigint NOT NULL, + *`c2` bigint DEFAULT NULL, + *PRIMARY KEY (`c1`))partition by range(`c1`)(partition p0 values less than(200), partition p1 values less than(500), partition p2 values less than(900)); + */ + + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_partition_aggregation", new String[] { "c1" }); + // aggregate without insert + ObTableAggregation obtableAggregation = client.aggregate("test_partition_aggregation"); + + obtableAggregation.addScanRange(new Object[] { 0L }, new Object[] { 150L }); + + // test + obtableAggregation.max("c2"); + obtableAggregation.min("c2"); + + // execute + ObTableAggregationResult obtableAggregationResult = obtableAggregation.execute(); + + // test null + Assert.assertNull(obtableAggregationResult.get("max(c2)")); + Assert.assertNull(obtableAggregationResult.get("min(c2)")); + } + + @Test + // Test aggregation exist null + public void testAggregationExistNull() throws Exception { + + /* + * CREATE TABLE `test_partition_aggregation` ( + *`c1` bigint NOT NULL, + *`c2` bigint DEFAULT NULL, + *PRIMARY KEY (`c1`))partition by range(`c1`)(partition p0 values less than(200), partition p1 values less than(500), partition p2 values less than(900)); + */ + + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_partition_aggregation", new String[] { "c1" }); + + // test with null + try { + client.insert("test_partition_aggregation", new Object[] { 50L }, + new String[] { "c2" }, new Object[] { null }); + client.insert("test_partition_aggregation", new Object[] { 100L }, + new String[] { "c2" }, new Object[] { null }); + + // with null + ObTableAggregation obtableAggregationWithoutFilter = client + .aggregate("test_partition_aggregation"); + + obtableAggregationWithoutFilter + .addScanRange(new Object[] { 0L }, new Object[] { 150L }); + + // test + obtableAggregationWithoutFilter.max("c1"); + obtableAggregationWithoutFilter.min("c1"); + obtableAggregationWithoutFilter.max("c2"); + obtableAggregationWithoutFilter.min("c2"); + + // execute + ObTableAggregationResult obtableAggregationResultWithoutFilter = obtableAggregationWithoutFilter + .execute(); + + // test with null + Assert.assertEquals(100L, obtableAggregationResultWithoutFilter.get("max(c1)")); + Assert.assertEquals(50L, obtableAggregationResultWithoutFilter.get("min(c1)")); + + Assert.assertEquals(null, obtableAggregationResultWithoutFilter.get("max(c2)")); + Assert.assertEquals(null, obtableAggregationResultWithoutFilter.get("min(c2)")); + + } finally { + client.delete("test_partition_aggregation", 50L); + client.delete("test_partition_aggregation", 100L); + } + } + + @Test + // Test aggregation with illegal column + public void testAggregationWithIllegalColumn() throws Exception { + + /* + * CREATE TABLE `test_partition_aggregation` ( + *`c1` bigint NOT NULL, + *`c2` bigint DEFAULT NULL, + *PRIMARY KEY (`c1`))partition by range(`c1`)(partition p0 values less than(200), partition p1 values less than(500), partition p2 values less than(900)); + */ + + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_partition_aggregation", new String[] { "c1" }); + + // test with null + try { + client.insert("test_partition_aggregation", new Object[] { 50L }, + new String[] { "c2" }, new Object[] { null }); + client.insert("test_partition_aggregation", new Object[] { 100L }, + new String[] { "c2" }, new Object[] { null }); + + // with + ObTableAggregation obtableAggregationWithIllegal = client + .aggregate("test_partition_aggregation"); + + obtableAggregationWithIllegal.addScanRange(new Object[] { 0L }, new Object[] { 150L }); + + // test illegal column + obtableAggregationWithIllegal.max("c3"); + + try { + obtableAggregationWithIllegal.execute(); + } catch (Exception e) { + System.out.printf("exception msg:%s\n", e.getMessage()); + Assert.assertTrue(e instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_ERR_UNEXPECTED.errorCode, + ((ObTableException) e).getErrorCode()); + } + + } finally { + client.delete("test_partition_aggregation", 50L); + client.delete("test_partition_aggregation", 100L); + } + } + + /* + CREATE TABLE test_aggregation ( + `c1` varchar(255), + `c2` int NOT NULL, + `c3` bigint NOT NULL, + `c4` float NOT NULL, + `c5` double NOT NULL, + `c6` tinyint NULL, + `c7` datetime, + PRIMARY KEY(`c1`) + ); + */ + + @Test + // Test aggregation with big int , should report error out of range + public void testAggregationWithBigint() throws Exception { + final String TABLE_NAME = "test_aggregation"; + try { + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement(TABLE_NAME, new String[] { "c1" }); + SimpleDateFormat sdf = new SimpleDateFormat(" yyyy-MM-dd HH:mm:ss "); + Date date1 = sdf.parse(" 1000-12-10 19:20:00 "); + Date date2 = sdf.parse(" 1010-07-10 19:20:00 "); + Date date3 = sdf.parse(" 1100-02-10 19:20:00 "); + + client.insert(TABLE_NAME, "first_row", new String[] { "c2", "c3", "c4", "c5", "c6", + "c7" }, new Object[] { 1, 9223372036854775807L, 1.0f, 1.0, (byte) 1, date1 }); + client.insert(TABLE_NAME, "second_row", new String[] { "c2", "c3", "c4", "c5", "c6", + "c7" }, new Object[] { 2, 9223372036854775807L, 2.0f, 2.0, (byte) 2, date2 }); + client.insert(TABLE_NAME, "third_row", new String[] { "c2", "c3", "c4", "c5", "c6", + "c7" }, new Object[] { 3, 9223372036854775807L, 3.0f, 3.0, (byte) 3, date3 }); + ObTableAggregation obtableAggregation = client.aggregate(TABLE_NAME); + + // test + obtableAggregation.sum("c3"); + + ObTableAggregationResult obtableAggregationResult = obtableAggregation.execute(); + + Assert.assertEquals(10L, obtableAggregationResult.get("sum(c3)")); + + } catch (Exception e) { + Assert.assertTrue(((ObTableException) e).getMessage().contains( + "[OB_DATA_OUT_OF_RANGE][Out of range value for column 'sum(c3)' at row 0]")); + } finally { + client.delete("test_aggregation", "first_row"); + client.delete("test_aggregation", "second_row"); + client.delete("test_aggregation", "third_row"); + } + } + + @Test + // Test aggregation with empty table + public void testAggregationEmptyVal() throws Exception { + + /* + * CREATE TABLE `test_partition_aggregation` ( + *`c1` bigint NOT NULL, + *`c2` bigint DEFAULT NULL, + *PRIMARY KEY (`c1`))partition by range(`c1`)(partition p0 values less than(200), partition p1 values less than(500), partition p2 values less than(900)); + */ + + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_partition_aggregation", new String[] { "c1" }); + + try { + // with filter + ObTableAggregation obtableAggregationWithFilter = client + .aggregate("test_partition_aggregation"); + + obtableAggregationWithFilter.addScanRange(new Object[] { 0L }, new Object[] { 150L }); + + // test + obtableAggregationWithFilter.max("c2"); + obtableAggregationWithFilter.min("c2"); + obtableAggregationWithFilter.count(); + obtableAggregationWithFilter.sum("c2"); + obtableAggregationWithFilter.avg("c2"); + + // filter + ObTableValueFilter filter = new ObTableValueFilter(ObCompareOp.GT, "c1", 90L); + + // add filter + obtableAggregationWithFilter.setFilter(filter); + + // execute + ObTableAggregationResult obtableAggregationResultWithFilter = obtableAggregationWithFilter + .execute(); + + // empty table generate null row + Assert.assertEquals(5, obtableAggregationResultWithFilter.getRow().size()); + Assert.assertEquals(null, obtableAggregationResultWithFilter.get("max(c2)")); + Assert.assertEquals(null, obtableAggregationResultWithFilter.get("min(c2)")); + Assert.assertEquals(null, obtableAggregationResultWithFilter.get("count()")); + Assert.assertEquals(null, obtableAggregationResultWithFilter.get("sum(c2)")); + Assert.assertEquals(null, obtableAggregationResultWithFilter.get("avg(c2)")); + + } finally { + } + } + +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableAuditTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableAuditTest.java new file mode 100644 index 00000000..2b95748e --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableAuditTest.java @@ -0,0 +1,608 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.filter.ObCompareOp; +import com.alipay.oceanbase.rpc.filter.ObTableFilterList; +import com.alipay.oceanbase.rpc.filter.ObTableValueFilter; +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateRequest; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.andList; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; + +public class ObTableAuditTest { + ObTableClient client; + public static String tableName = "audit_test"; + + @Before + public void setup() throws Exception { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { "c1" }); + } + + private static void setMinimalImage() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("SET GLOBAL binlog_row_image=MINIMAL"); + } + + private static void setFullImage() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("SET GLOBAL binlog_row_image=Full"); + } + + public static String generateRandomString(int length) { + String characters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; + StringBuilder sb = new StringBuilder(); + + Random random = new Random(); + for (int i = 0; i < length; i++) { + int index = random.nextInt(characters.length()); + char randomChar = characters.charAt(index); + sb.append(randomChar); + } + + return sb.toString(); + } + + public static int generateRandomNumber() { + Random random = new Random(); + return random.nextInt(64536) + 1000; + } + + private List get_sql_id(String keyWord) throws SQLException { + List sqlIds = new ArrayList<>(); + String tenantName = ObTableClientTestUtil.getTenantName(); + Connection conn = ObTableClientTestUtil.getSysConnection(); + PreparedStatement ps = conn.prepareStatement("select sql_id from oceanbase.__all_virtual_sql_audit where query_sql like " + + "\"%" + keyWord + "%\"" + "and tenant_name=" + "\"" + tenantName + "\"" + "order by request_time asc" + ";"); + ResultSet rs = ps.executeQuery(); + + while (rs.next()) { + String sqlId = rs.getString(1); + sqlIds.add(sqlId); + } + + for (String sqlId : sqlIds) { + System.out.println(sqlId); + } + + return sqlIds; + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testSingleOperation() throws Exception { + try { + String prefix = generateRandomString(10); + client.insertOrUpdate(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", prefix)).execute(); + List sqlIds = get_sql_id(prefix); + Assert.assertEquals(1, sqlIds.size()); + + // same operation and columns generate same sql_id + sqlIds.clear(); + client.insertOrUpdate(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", prefix)).execute(); + sqlIds = get_sql_id(prefix); + Assert.assertEquals(2, sqlIds.size()); + for (String sqlId : sqlIds) { + Assert.assertEquals(sqlIds.get(0), sqlId); + } + + // different operation generate different sql_id + sqlIds.clear(); + client.update(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", prefix)).execute(); + sqlIds = get_sql_id(prefix); + Assert.assertEquals(3, sqlIds.size()); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(1)); + Assert.assertNotEquals(sqlIds.get(1), sqlIds.get(2)); + + // different columns generate different sql_id + // write c3 + sqlIds.clear(); + client.update(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c3", prefix)).execute(); + sqlIds = get_sql_id(prefix); + Assert.assertEquals(4, sqlIds.size()); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(1)); + Assert.assertNotEquals(sqlIds.get(1), sqlIds.get(2)); + Assert.assertNotEquals(sqlIds.get(2), sqlIds.get(3)); + } finally { + client.delete(tableName, 1L); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testMultiOperation() throws Exception { + try { + String prefix = generateRandomString(10); + BatchOperation batchOps = client.batchOperation(tableName); + Insert ins1 = client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", prefix)); + Insert ins2 = client.insert(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", prefix)); + batchOps.addOperation(ins1, ins2).execute(); + List sqlIds = get_sql_id(prefix); + Assert.assertEquals(1, sqlIds.size()); + + // same sql_id even id rowkey value is different + sqlIds.clear(); + batchOps = client.batchOperation(tableName); + ins1 = client.insert(tableName).setRowKey(colVal("c1", 3L)) + .addMutateColVal(colVal("c2", prefix)); + ins2 = client.insert(tableName).setRowKey(colVal("c1", 4L)) + .addMutateColVal(colVal("c2", prefix)); + batchOps.addOperation(ins1, ins2).execute(); + sqlIds = get_sql_id(prefix); + Assert.assertEquals(2, sqlIds.size()); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(1)); + } finally { + client.delete(tableName, 1L); + client.delete(tableName, 2L); + client.delete(tableName, 3L); + client.delete(tableName, 4L); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testMixedBatchOperation1() throws Exception { + try { + String prefix = generateRandomString(10); + // mixed op has multi sql_id + BatchOperation batchOps = client.batchOperation(tableName); + Insert ins = client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", prefix)); + InsertOrUpdate insUp = client.insertOrUpdate(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", prefix)); + batchOps.addOperation(ins, insUp).execute(); + List sqlIds = get_sql_id(prefix); + Assert.assertEquals(2, sqlIds.size()); + } finally { + client.delete(tableName, 1L); + client.delete(tableName, 2L); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testMixedBatchOperation2() throws Exception { + try { + String prefix = generateRandomString(10); + // mixed op has multi sql_id + BatchOperation batchOps = client.batchOperation(tableName); + Insert ins = client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", prefix)); + InsertOrUpdate insUp = client.insertOrUpdate(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", prefix)); + Append appn = client.append(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", prefix)); + batchOps.addOperation(ins, insUp, appn).execute(); + List sqlIds = get_sql_id(prefix); + Assert.assertEquals(3, sqlIds.size()); + } finally { + client.delete(tableName, 1L); + client.delete(tableName, 2L); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testSyncQuery() throws Exception { + // query $table_name $column_0, $column_1, ..., $column_n range:$column_0, $column_1, ..., $column_n index:$index_name $filter + try { + // insert + client.insertOrUpdate(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", "c2_val")).execute(); + + // query + int limit = generateRandomNumber(); + TableQuery tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1"); + tableQuery.limit(limit); + tableQuery.execute(); + List sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(1, sqlIds.size()); + + // different Range generate same sql_id + // 0 - 100 + sqlIds.clear(); + tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 100L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1"); + tableQuery.limit(limit); + tableQuery.execute(); + sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(2, sqlIds.size()); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(1)); + + // different select columns generate different sql_id + // select c1, c2 + sqlIds.clear(); + tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1", "c2"); + tableQuery.limit(limit); + tableQuery.execute(); + sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(3, sqlIds.size()); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(1)); + Assert.assertNotEquals(sqlIds.get(1), sqlIds.get(2)); + + // different index generate different sql_id + // index idx_c2 + sqlIds.clear(); + tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { "c2" }, new Object[] { "c2_val" }); + tableQuery.setScanRangeColumns("c2"); + tableQuery.select("c1", "c2"); + tableQuery.limit(limit); + tableQuery.indexName("idx_c2"); + tableQuery.execute(); + sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(4, sqlIds.size()); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(1)); + Assert.assertNotEquals(sqlIds.get(1), sqlIds.get(2)); + Assert.assertNotEquals(sqlIds.get(2), sqlIds.get(3)); + } finally { + client.delete(tableName, 1L); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testSyncQueryIndex() throws Exception { + // query $table_name $column_0, $column_1, ..., $column_n range:$column_0, $column_1, ..., $column_n index:$index_name $filter + try { + // different index generate different sql_id + int limit = generateRandomNumber(); + client.query(tableName).select("c1").indexName("idx_c2").limit(limit).execute(); + client.query(tableName).select("c1").indexName("idx_c3").limit(limit).execute(); + List sqlIds = get_sql_id("limit:" + limit); + Assert.assertEquals(2, sqlIds.size()); + Assert.assertNotEquals(sqlIds.get(0), sqlIds.get(1)); + } finally { + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testSyncQueryFilter1() throws Exception { + // query $table_name $column_0, $column_1, ..., $column_n range:$column_0, $column_1, ..., $column_n index:$index_name $filter + try { + // different filter column generate different sql_id + int limit = generateRandomNumber(); + ObTableValueFilter filter1 = new ObTableValueFilter(ObCompareOp.EQ, "c2", "hello"); + ObTableValueFilter filter2 = new ObTableValueFilter(ObCompareOp.EQ, "c3", "hello"); + client.query(tableName).select("c1").setFilter(filter1).limit(limit).execute(); + client.query(tableName).select("c1").setFilter(filter2).limit(limit).execute(); + List sqlIds = get_sql_id("limit:" + limit); + Assert.assertEquals(2, sqlIds.size()); + Assert.assertNotEquals(sqlIds.get(0), sqlIds.get(1)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testSyncQueryFilter2() throws Exception { + // query $table_name $column_0, $column_1, ..., $column_n range:$column_0, $column_1, ..., $column_n index:$index_name $filter + try { + int limit = generateRandomNumber(); + ObTableValueFilter filter1 = new ObTableValueFilter(ObCompareOp.EQ, "c2", "hello"); + ObTableValueFilter filter2 = new ObTableValueFilter(ObCompareOp.EQ, "c3", "hello"); + ObTableFilterList list1 = andList(filter1, filter2); + ObTableFilterList list2 = andList(filter2, filter1); + client.query(tableName).select("c1").setFilter(list1).limit(limit).execute(); + client.query(tableName).select("c1").setFilter(list2).limit(limit).execute(); + List sqlIds = get_sql_id("limit:" + limit); + Assert.assertEquals(2, sqlIds.size()); + Assert.assertNotEquals(sqlIds.get(0), sqlIds.get(1)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testSyncQueryFilter3() throws Exception { + // query $table_name $column_0, $column_1, ..., $column_n range:$column_0, $column_1, ..., $column_n index:$index_name $filter + try { + // insert + client.insertOrUpdate(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", "c2_val")).execute(); + + // query + int limit = generateRandomNumber(); + TableQuery tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1"); + tableQuery.limit(limit); + tableQuery.execute(); + List sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(1, sqlIds.size()); + + // has filter generate different sql_id + ObTableValueFilter filter = new ObTableValueFilter(ObCompareOp.GT, "c2", "c2_val"); + sqlIds.clear(); + tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 100L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1"); + tableQuery.limit(limit); + tableQuery.setFilter(filter); + tableQuery.execute(); + sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(2, sqlIds.size()); + Assert.assertNotEquals(sqlIds.get(0), sqlIds.get(1)); + } finally { + client.delete(tableName, 1L); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testAsyncQuery() throws Exception { + // query $table_name $column_0, $column_1, ..., $column_n range:$column_0, $column_1, ..., $column_n index:$index_name $filter + try { + // insert + client.insertOrUpdate(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", "c2_val")).execute(); + + // query + int limit = generateRandomNumber(); + TableQuery tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1"); + tableQuery.limit(limit); + tableQuery.asyncExecute(); + List sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(1, sqlIds.size()); + + // different Range generate same sql_id + // 0 - 100 + sqlIds.clear(); + tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 100L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1"); + tableQuery.limit(limit); + tableQuery.asyncExecute(); + sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(2, sqlIds.size()); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(1)); + + // different select columns generate different sql_id + // select c1, c2 + sqlIds.clear(); + tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1", "c2"); + tableQuery.limit(limit); + tableQuery.asyncExecute(); + sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(3, sqlIds.size()); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(1)); + Assert.assertNotEquals(sqlIds.get(1), sqlIds.get(2)); + + // different index generate different sql_id + // index idx_c2 + sqlIds.clear(); + tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { "c2" }, new Object[] { "c2_val" }); + tableQuery.setScanRangeColumns("c2"); + tableQuery.select("c1", "c2"); + tableQuery.limit(limit); + tableQuery.indexName("idx_c2"); + tableQuery.asyncExecute(); + sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(4, sqlIds.size()); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(1)); + Assert.assertNotEquals(sqlIds.get(1), sqlIds.get(2)); + Assert.assertNotEquals(sqlIds.get(2), sqlIds.get(3)); + } finally { + client.delete(tableName, 1L); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testAsyncQueryFilter() throws Exception { + // query $table_name $column_0, $column_1, ..., $column_n range:$column_0, $column_1, ..., $column_n index:$index_name $filter + try { + // insert + client.insertOrUpdate(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", "c2_val")).execute(); + + // query + int limit = generateRandomNumber(); + TableQuery tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1"); + tableQuery.limit(limit); + tableQuery.execute(); + List sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(1, sqlIds.size()); + + // has filter generate different sql_id + ObTableValueFilter filter = new ObTableValueFilter(ObCompareOp.GT, "c2", "c2_val"); + sqlIds.clear(); + tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 100L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1"); + tableQuery.limit(limit); + tableQuery.setFilter(filter); + tableQuery.execute(); + sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(2, sqlIds.size()); + Assert.assertNotEquals(sqlIds.get(0), sqlIds.get(1)); + } finally { + client.delete(tableName, 1L); + } + } + + /** + CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + **/ + @Test + public void testQueryAndMutate() throws Exception { + try { + // insert + client.insertOrUpdate(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", "c2_val")).execute(); + + int limit = generateRandomNumber(); + TableQuery tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.setScanRangeColumns("c1"); + tableQuery.select("c1"); + tableQuery.limit(limit); + ObTableQueryAndMutateRequest req = client.obTableQueryAndAppend(tableQuery, + new String[] { "c2" }, new Object[] { "_append0" }, false); + client.execute(req); + List sqlIds = get_sql_id("limit:" + String.valueOf(limit)); + Assert.assertEquals(3, sqlIds.size()); // query twice + Assert.assertNotEquals(sqlIds.get(0), sqlIds.get(1)); + Assert.assertEquals(sqlIds.get(0), sqlIds.get(2)); + } finally { + client.delete(tableName, 1L); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableCheckAndInsUpTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableCheckAndInsUpTest.java new file mode 100644 index 00000000..9226b03e --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableCheckAndInsUpTest.java @@ -0,0 +1,537 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.checkandmutate.CheckAndInsUp; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.filter.ObCompareOp; +import com.alipay.oceanbase.rpc.filter.ObTableFilter; +import com.alipay.oceanbase.rpc.mutation.BatchOperation; +import com.alipay.oceanbase.rpc.mutation.Delete; +import com.alipay.oceanbase.rpc.mutation.InsertOrUpdate; +import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import com.alipay.oceanbase.rpc.util.TimeUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Timestamp; +import java.util.Date; +import java.util.Map; + +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.compareVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.row; + +public class ObTableCheckAndInsUpTest { + public ObTableClient client; + private static long MINI_SUPP_VERSION = ObGlobal.calcVersion(4, (short) 2, (byte) 1, + (byte) 2); + private static final String TABLE_NAME = "test_mutation"; + + @Before + public void setup() throws Exception { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + } + + private boolean isVersionSupported() { + if (ObTableClientTestUtil.isOBVersionGreaterEqualThan(MINI_SUPP_VERSION)) { + return true; + } + return false; + } + + /* + CREATE TABLE `test_mutation` ( + `c1` bigint NOT NULL, + `c2` varchar(20) NOT NULL, + `c3` varbinary(1024) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`c1`, `c2`)) partition by range columns (`c1`) ( + PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), + PARTITION p2 VALUES LESS THAN MAXVALUE); + + */ + @Test + public void testBatchWithDiffRows() throws Exception { + if (!isVersionSupported()) { + System.out.println("current version is not supported, current version: " + + ObGlobal.OB_VERSION); + return; + } + try { + // 0. prepare data, insert(1, 'c2_v0', 'c3_v0', 100),(2, 'c2_v0', 'c3_v0', 100),(3, 'c2_v0', 'c3_v0', 100),(4, 'c2_v0', 'c3_v0', 100) + for (long i = 1L; i <= 4L; i++) { + InsertOrUpdate insertOrUpdate = client.insertOrUpdate(TABLE_NAME); + insertOrUpdate.setRowKey(row(colVal("c1", i), colVal("c2", "c2_v0"))); + insertOrUpdate.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 100L))); + MutationResult res = insertOrUpdate.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + // 1. check exists match: insup(1, 'c2_v0', 'c3_v0', 200) if exists c3 >= 'c3_v0'; + InsertOrUpdate insertOrUpdate1 = new InsertOrUpdate(); + insertOrUpdate1.setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_v0"))); + insertOrUpdate1.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 200L))); + ObTableFilter filter = compareVal(ObCompareOp.GE, "c3", "c3_v0"); + CheckAndInsUp checkAndInsUp1 = new CheckAndInsUp(filter, insertOrUpdate1, true); + + // 2. check exists not match: insup(2, 'c2_v0', 'c3_v0', 200) if exists c3 > 'c3_v0'; + InsertOrUpdate insertOrUpdate2 = new InsertOrUpdate(); + insertOrUpdate2.setRowKey(row(colVal("c1", 2L), colVal("c2", "c2_v0"))); + insertOrUpdate2.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 200L))); + filter = compareVal(ObCompareOp.GT, "c3", "c3_v0"); + CheckAndInsUp checkAndInsUp2 = new CheckAndInsUp(filter, insertOrUpdate2, true); + + // 3. check no exists match: insup(3, 'c2_v0', 'c3_v0', 200) if not exists c4 > 200 + InsertOrUpdate insertOrUpdate3 = new InsertOrUpdate(); + insertOrUpdate3.setRowKey(row(colVal("c1", 3L), colVal("c2", "c2_v0"))); + insertOrUpdate3.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 200L))); + filter = compareVal(ObCompareOp.GE, "c4", 200L); + CheckAndInsUp checkAndInsUp3 = new CheckAndInsUp(filter, insertOrUpdate3, false); + + // 4. check no exists not match: insup(4, 'c2_v0', 'c3_v0', 200) if exists c4 is null + InsertOrUpdate insertOrUpdate4 = new InsertOrUpdate(); + insertOrUpdate4.setRowKey(row(colVal("c1", 4L), colVal("c2", "c2_v0"))); + insertOrUpdate4.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 200L))); + filter = compareVal(ObCompareOp.IS_NOT, "c4", null); + CheckAndInsUp checkAndInsUp4 = new CheckAndInsUp(filter, insertOrUpdate4, false); + + // 5. verify result + batchOperation.addOperation(checkAndInsUp1, checkAndInsUp2, checkAndInsUp3, + checkAndInsUp4); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(4, batchOperationResult.size()); + Assert.assertEquals(1, batchOperationResult.get(0).getAffectedRows()); + Assert.assertEquals(0, batchOperationResult.get(1).getAffectedRows()); + Assert.assertEquals(1, batchOperationResult.get(2).getAffectedRows()); + Assert.assertEquals(0, batchOperationResult.get(3).getAffectedRows()); + + Map res = client.get(TABLE_NAME, new Object[] { 1L, "c2_v0" }, + new String[] { "c4" }); + Assert.assertEquals(200L, res.get("c4")); + res = client.get(TABLE_NAME, new Object[] { 2L, "c2_v0" }, new String[] { "c4" }); + Assert.assertEquals(100L, res.get("c4")); + res = client.get(TABLE_NAME, new Object[] { 3L, "c2_v0" }, new String[] { "c4" }); + Assert.assertEquals(200L, res.get("c4")); + res = client.get(TABLE_NAME, new Object[] { 4L, "c2_v0" }, new String[] { "c4" }); + Assert.assertEquals(100L, res.get("c4")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (long i = 1L; i <= 4L; i++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", i), colVal("c2", "c2_v0"))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchWithSameRows() throws Exception { + if (!isVersionSupported()) { + System.out.println("current version is not supported, current version: " + + ObGlobal.OB_VERSION); + return; + } + + try { + // 0. prepare data, insert(5, 'c2_v0', 'c3_v0', 100) + InsertOrUpdate insertOrUpdate = client.insertOrUpdate(TABLE_NAME); + insertOrUpdate.setRowKey(row(colVal("c1", 5L), colVal("c2", "c2_v0"))); + insertOrUpdate.addMutateRow(row(colVal("c3", "c3_v0".getBytes()), colVal("c4", 100L))); + MutationResult result = insertOrUpdate.execute(); + Assert.assertEquals(1, result.getAffectedRows()); + + // 1. check exists match: insup(5, 'c2_v0', c3_v0, 200) if exists c3 is not null; + InsertOrUpdate insertOrUpdate1 = new InsertOrUpdate(); + insertOrUpdate1.setRowKey(row(colVal("c1", 5L), colVal("c2", "c2_v0"))); + insertOrUpdate1.addMutateRow(row(colVal("c4", 200L))); + ObTableFilter filter = compareVal(ObCompareOp.IS_NOT, "c3", null); + CheckAndInsUp checkAndInsUp1 = new CheckAndInsUp(filter, insertOrUpdate1, true); + + // 2. check exists not match: insup(5, 'c2_v0', 'c3_v1', 200) if exists c4 > 200 ; + InsertOrUpdate insertOrUpdate2 = new InsertOrUpdate(); + insertOrUpdate2.setRowKey(row(colVal("c1", 5L), colVal("c2", "c2_v0"))); + insertOrUpdate2.addMutateRow(row(colVal("c3", "c3_v1".getBytes()), colVal("c4", 200L))); + filter = compareVal(ObCompareOp.GT, "c4", 200L); + CheckAndInsUp checkAndInsUp2 = new CheckAndInsUp(filter, insertOrUpdate2, true); + + // 3. check no exists match: insup(5, 'c2_v0', 'c3_v1', 300) if not exists c4 > 300 ; + InsertOrUpdate insertOrUpdate3 = new InsertOrUpdate(); + insertOrUpdate3.setRowKey(row(colVal("c1", 5L), colVal("c2", "c2_v0"))); + insertOrUpdate3.addMutateRow(row(colVal("c3", "c3_v1".getBytes()), colVal("c4", 300L))); + filter = compareVal(ObCompareOp.GT, "c4", 300L); + CheckAndInsUp checkAndInsUp3 = new CheckAndInsUp(filter, insertOrUpdate3, false); + + // 3. check no exists not match: insup(5, 'c2_v0', 'c3_v1', 400) if not exists c4 >= 300 ; + InsertOrUpdate insertOrUpdate4 = new InsertOrUpdate(); + insertOrUpdate4.setRowKey(row(colVal("c1", 5L), colVal("c2", "c2_v0"))); + insertOrUpdate4.addMutateRow(row(colVal("c3", "c3_v1".getBytes()), colVal("c4", 400L))); + filter = compareVal(ObCompareOp.GE, "c4", 300L); + CheckAndInsUp checkAndInsUp4 = new CheckAndInsUp(filter, insertOrUpdate4, false); + + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + batchOperation.addOperation(checkAndInsUp1, checkAndInsUp2, checkAndInsUp3, + checkAndInsUp4); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(4, batchOperationResult.size()); + Assert.assertEquals(1, batchOperationResult.get(0).getAffectedRows()); + Assert.assertEquals(0, batchOperationResult.get(1).getAffectedRows()); + Assert.assertEquals(1, batchOperationResult.get(2).getAffectedRows()); + Assert.assertEquals(0, batchOperationResult.get(3).getAffectedRows()); + + Map res = client.get(TABLE_NAME, new Object[] { 5L, "c2_v0" }, null); + Assert.assertEquals("c3_v1", new String((byte[]) res.get("c3"), "UTF-8")); + Assert.assertEquals(300L, res.get("c4")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", 5L), colVal("c2", "c2_v0"))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + + @Test + public void testBatchWithReverseRowKwyColumn() throws Exception { + if (!isVersionSupported()) { + System.out.println("current version is not supported, current version: " + + ObGlobal.OB_VERSION); + return; + } + String testTable = "test_mutation_column_reverse"; + + try { + // 1. check exists match: insup (c2, c1, c3, c4) (5, 'c2_v0', c3_v0, 100) if not exists c3 is not null; + InsertOrUpdate insertOrUpdate1 = new InsertOrUpdate(); + insertOrUpdate1.setRowKey(row(colVal("c2", 5L), colVal("c1", "c2_v0"))); + insertOrUpdate1.addMutateRow(row(colVal("c4", 100L))); + ObTableFilter filter = compareVal(ObCompareOp.IS_NOT, "c3", null); + CheckAndInsUp checkAndInsUp1 = new CheckAndInsUp(filter, insertOrUpdate1, false); + + // 2. check exists not match: insup (c2, c1, c3, c4) (5, 'c2_v0', 'c3_v1', 200) if exists c4 < 200 ; + InsertOrUpdate insertOrUpdate2 = new InsertOrUpdate(); + insertOrUpdate2.setRowKey(row(colVal("c2", 5L), colVal("c1", "c2_v0"))); + insertOrUpdate2.addMutateRow(row(colVal("c3", "c3_v1".getBytes()), colVal("c4", 200L))); + filter = compareVal(ObCompareOp.LT, "c4", 200L); + CheckAndInsUp checkAndInsUp2 = new CheckAndInsUp(filter, insertOrUpdate2, true); + + BatchOperation batchOperation = client.batchOperation(testTable); + batchOperation.addOperation(checkAndInsUp1, checkAndInsUp2); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(2, batchOperationResult.size()); + Assert.assertEquals(1, batchOperationResult.get(0).getAffectedRows()); + Assert.assertEquals(1, batchOperationResult.get(1).getAffectedRows()); + + Map res = client.get(testTable, new Object[] { 5L, "c2_v0" }, null); + Assert.assertEquals("c3_v1", new String((byte[]) res.get("c3"), "UTF-8")); + Assert.assertEquals(200L, res.get("c4")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + Delete delete = client.delete(testTable); + delete.setRowKey(row(colVal("c1", 5L), colVal("c2", "c2_v0"))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + + @Test + public void testSingleCheckInsUp() throws Exception { + if (!isVersionSupported()) { + System.out.println("current version is not supported, current version: " + + ObGlobal.OB_VERSION); + return; + } + + try { + // 0. prepare data, insert(1, 'c2_v0', 'c3_v0', 100),(2, 'c2_v0', 'c3_v0', 100),(3, 'c2_v0', 'c3_v0', 100),(4, 'c2_v0', 'c3_v0', 100) + for (long i = 1L; i <= 4L; i++) { + InsertOrUpdate insertOrUpdate = client.insertOrUpdate(TABLE_NAME); + insertOrUpdate.setRowKey(row(colVal("c1", i), colVal("c2", "c2_v0"))); + insertOrUpdate.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 100L))); + MutationResult res = insertOrUpdate.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + + // 1. check exists match: insup(1, 'c2_v0', 'c3_v0', 200) if exists c3 >= 'c3_v0'; + InsertOrUpdate insertOrUpdate1 = new InsertOrUpdate(); + insertOrUpdate1.setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_v0"))); + insertOrUpdate1.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 200L))); + ObTableFilter filter = compareVal(ObCompareOp.GE, "c3", "c3_v0"); + CheckAndInsUp checkAndInsUp1 = client.checkAndInsUp(TABLE_NAME, filter, + insertOrUpdate1, true); + MutationResult result1 = checkAndInsUp1.execute(); + Assert.assertEquals(1, result1.getAffectedRows()); + Map res = client.get(TABLE_NAME, new Object[] { 1L, "c2_v0" }, + new String[] { "c3", "c4" }); + Assert.assertEquals("c3_v0", new String((byte[]) res.get("c3"), "UTF-8")); + Assert.assertEquals(200L, res.get("c4")); + + // 2. check exists not match: insup(2, 'c2_v0', 'c3_v0', 200) if exists c3 > 'c3_v0'; + InsertOrUpdate insertOrUpdate2 = new InsertOrUpdate(); + insertOrUpdate2.setRowKey(row(colVal("c1", 2L), colVal("c2", "c2_v0"))); + insertOrUpdate2.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 200L))); + filter = compareVal(ObCompareOp.GT, "c3", "c3_v0"); + CheckAndInsUp checkAndInsUp2 = client.checkAndInsUp(TABLE_NAME, filter, + insertOrUpdate2, true); + MutationResult result2 = checkAndInsUp2.execute(); + Assert.assertEquals(0, result2.getAffectedRows()); + res = client.get(TABLE_NAME, new Object[] { 2L, "c2_v0" }, new String[] { "c3", "c4" }); + Assert.assertEquals("c3_v0", new String((byte[]) res.get("c3"), "UTF-8")); + Assert.assertEquals(100L, res.get("c4")); + + // 3. check no exists match: insup(3, 'c2_v0', 'c3_v0', 200) if not exists c4 > 200 + InsertOrUpdate insertOrUpdate3 = new InsertOrUpdate(); + insertOrUpdate3.setRowKey(row(colVal("c1", 3L), colVal("c2", "c2_v0"))); + insertOrUpdate3.addMutateRow(row(colVal("c3", "c3_v1"), colVal("c4", 200L))); + filter = compareVal(ObCompareOp.GE, "c4", 200L); + CheckAndInsUp checkAndInsUp3 = client.checkAndInsUp(TABLE_NAME, filter, + insertOrUpdate3, false); + MutationResult result3 = checkAndInsUp3.execute(); + Assert.assertEquals(1, result3.getAffectedRows()); + res = client.get(TABLE_NAME, new Object[] { 3L, "c2_v0" }, new String[] { "c3", "c4" }); + Assert.assertEquals("c3_v1", new String((byte[]) res.get("c3"), "UTF-8")); + Assert.assertEquals(200L, res.get("c4")); + + // 4. check no exists not match: insup(4, 'c2_v0', 'c3_v0', 200) if exists c4 is null + InsertOrUpdate insertOrUpdate4 = new InsertOrUpdate(); + insertOrUpdate4.setRowKey(row(colVal("c1", 4L), colVal("c2", "c2_v0"))); + insertOrUpdate4.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 200L))); + filter = compareVal(ObCompareOp.IS_NOT, "c4", null); + CheckAndInsUp checkAndInsUp4 = client.checkAndInsUp(TABLE_NAME, filter, + insertOrUpdate4, false); + MutationResult result4 = checkAndInsUp4.execute(); + Assert.assertEquals(0, result4.getAffectedRows()); + res = client.get(TABLE_NAME, new Object[] { 3L, "c2_v0" }, new String[] { "c3", "c4" }); + Assert.assertEquals("c3_v1", new String((byte[]) res.get("c3"), "UTF-8")); + Assert.assertEquals(200L, res.get("c4")); + + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (long i = 1L; i <= 4L; i++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", i), colVal("c2", "c2_v0"))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchCheckInsUpMutPart() throws Exception { + if (!isVersionSupported()) { + System.out.println("current version is not supported, current version: " + + ObGlobal.OB_VERSION); + return; + } + // insert two record in different partition + // insert (100, "c2_val", "c3_val", 100) + InsertOrUpdate insertOrUpdate = client.insertOrUpdate(TABLE_NAME); + insertOrUpdate.setRowKey(row(colVal("c1", 100L), colVal("c2", "c2_val"))); + insertOrUpdate.addMutateRow(row(colVal("c3", "c3_v0"), colVal("c4", 100L))); + MutationResult res = insertOrUpdate.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + // insert (400, "c2_val", "c3_val", 400) + insertOrUpdate = client.insertOrUpdate(TABLE_NAME); + insertOrUpdate.setRowKey(row(colVal("c1", 400L), colVal("c2", "c2_val"))); + insertOrUpdate.addMutateRow(row(colVal("c3", "c3_val"), colVal("c4", 400L))); + res = insertOrUpdate.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + + try { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + // 1. check exists not match: insup(100, 'c2_val', 'c3_val', 200) if exists c4 > 100 + InsertOrUpdate insertOrUpdate1 = new InsertOrUpdate(); + insertOrUpdate1.setRowKey(row(colVal("c1", 100L), colVal("c2", "c2_val"))); + insertOrUpdate1.addMutateRow(row(colVal("c3", "c3_val"), colVal("c4", 200L))); + ObTableFilter filter = compareVal(ObCompareOp.GT, "c4", 100L); + CheckAndInsUp checkAndInsUp1 = new CheckAndInsUp(filter, insertOrUpdate1, true); + // 2. check not exists match: insup(400, 'c2_val', 'c3_val', 500) if not exists c4 >= 500 + InsertOrUpdate insertOrUpdate2 = new InsertOrUpdate(); + insertOrUpdate2.setRowKey(row(colVal("c1", 400L), colVal("c2", "c2_val"))); + insertOrUpdate2.addMutateRow(row(colVal("c3", "c3_val"), colVal("c4", 500L))); + ObTableFilter filter2 = compareVal(ObCompareOp.GE, "c4", 500L); + CheckAndInsUp checkAndInsUp2 = new CheckAndInsUp(filter2, insertOrUpdate2, false); + + // 3. execute batch + batchOperation.addOperation(checkAndInsUp1, checkAndInsUp2); + BatchOperationResult result = batchOperation.execute(); + Assert.assertEquals(2, result.getCorrectCount()); + Assert.assertEquals(0, result.get(0).getAffectedRows()); + Assert.assertEquals(1, result.get(1).getAffectedRows()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } + } + + /* + CREATE TABLE IF NOT EXISTS `test_bigint_table` ( + `c1` bigint(20) NOT NULL, + `c2` bigint(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + */ + @Test + public void testNonPartCheckAndInsUp() throws Exception { + String tableName = "test_bigint_table"; + if (!isVersionSupported()) { + System.out.println("current version is not supported, current version: " + + ObGlobal.OB_VERSION); + return; + } + // pre-clean data + client.delete(tableName).addScanRange(ObObj.getMin(), ObObj.getMax()).execute(); + try { + // 1. single operation + // check not exists match: insup(1, 100) if not exists c2 <= 100 + InsertOrUpdate insertOrUpdate1 = new InsertOrUpdate(); + insertOrUpdate1.setRowKey(row(colVal("c1", 1L))); + insertOrUpdate1.addMutateRow(row(colVal("c2", 100L))); + ObTableFilter filter1 = compareVal(ObCompareOp.GE, "c2", 100L); + CheckAndInsUp checkAndInsUp1 = client.checkAndInsUp(tableName, filter1, + insertOrUpdate1, false); + MutationResult result1 = checkAndInsUp1.execute(); + Assert.assertEquals(1, result1.getAffectedRows()); + + // 2. batch oepration + BatchOperation batchOperation = client.batchOperation(tableName); + // 2.1 check exists match: insup(1, 200) if exists c2 < 100 + InsertOrUpdate insertOrUpdate2 = new InsertOrUpdate(); + insertOrUpdate2.setRowKey(row(colVal("c1", 1L))); + insertOrUpdate2.addMutateRow(row(colVal("c2", 200L))); + ObTableFilter filter2 = compareVal(ObCompareOp.LT, "c2", 200L); + CheckAndInsUp checkAndInsUp2 = new CheckAndInsUp(filter2, insertOrUpdate2, true); + + // 2.2 check not exists match: insup(2, 200) if not exists c2 >= 200 + InsertOrUpdate insertOrUpdate3 = new InsertOrUpdate(); + insertOrUpdate3.setRowKey(row(colVal("c1", 2L))); + insertOrUpdate3.addMutateRow(row(colVal("c2", 200L))); + ObTableFilter filter3 = compareVal(ObCompareOp.GE, "c2", 200L); + CheckAndInsUp checkAndInsUp3 = new CheckAndInsUp(filter3, insertOrUpdate3, false); + + // 3. execute batch + batchOperation.addOperation(checkAndInsUp2, checkAndInsUp3); + BatchOperationResult result = batchOperation.execute(); + Assert.assertEquals(2, result.getCorrectCount()); + Assert.assertEquals(1, result.get(0).getAffectedRows()); + Assert.assertEquals(1, result.get(1).getAffectedRows()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + } + } + + /* + CREATE TABLE IF NOT EXISTS `test_table_object` ( + `c1` tinyint primary key, + `c2` smallint not null, + `c3` int not null, + `c4` bigint not null, + `c5` varchar(128) not null, + `c6` varbinary(128) not null, + `c7` float not null, + `c8` double not null, + `c9` timestamp not null, + `c10` datetime not null + ); + */ + @Test + public void testCheckAndInsUpWithDiffObj() throws Exception { + String tableName = "test_table_object"; + if (!isVersionSupported()) { + System.out.println("current version is not supported, current version: " + + ObGlobal.OB_VERSION); + return; + } + // pre-clean data + client.delete(tableName).addScanRange(ObObj.getMin(), ObObj.getMax()).execute(); + + // 1. check not exists match: insup(1, 1, 1, 1, "hello", "world", 1.0f, 1.0d, now(), now()) if not exists c3 >= 200 + Byte c1Val = 1; + short c2Val = 1; + int c3Val = 1; + long c4Val = 1; + String c5Val = "hello"; + byte[] c6Val = "world".getBytes(); + float c7Val = 1.0f; + double c8Val = 1.0d; + long timeInMillis = System.currentTimeMillis(); + Timestamp c9Val = new Timestamp(timeInMillis); + Date c10Val = TimeUtils.strToDate("2024-01-30"); + Object c11Val = null; + + Object[] values = { c2Val, c3Val, c4Val, c5Val, c6Val, c7Val, c8Val, c9Val, c10Val, c11Val }; + String[] columns = { "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", "c11" }; + + InsertOrUpdate insertOrUpdate1 = new InsertOrUpdate(); + insertOrUpdate1.setRowKey(row(colVal("c1", c1Val))); + for (int i = 0; i < values.length; i++) { + insertOrUpdate1.addMutateRow(row(colVal(columns[i], values[i]))); + } + ObTableFilter filter1 = compareVal(ObCompareOp.GE, "c3", 200); + CheckAndInsUp checkAndInsUp1 = new CheckAndInsUp(filter1, insertOrUpdate1, false); + BatchOperation batchOperation = client.batchOperation(tableName); + batchOperation.addOperation(checkAndInsUp1); + BatchOperationResult result = batchOperation.execute(); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(1, result.get(0).getAffectedRows()); + + // 2. check not exists: insup(c1, c2, c3) (2, min, max) if not exists c3 >= 200 + // just test the table object deserialize + try { + Object[] values2 = { ObObj.getMin(), ObObj.getMax() }; + String[] columns2 = { "c2", "c3" }; + c1Val = 2; + + InsertOrUpdate insertOrUpdate2 = new InsertOrUpdate(); + insertOrUpdate2.setRowKey(row(colVal("c1", c1Val))); + for (int i = 1; i < values2.length; i++) { + insertOrUpdate2.addMutateRow(row(colVal(columns2[i], values2[i]))); + } + ObTableFilter filter2 = compareVal(ObCompareOp.GE, "c3", 200); + CheckAndInsUp checkAndInsUp2 = new CheckAndInsUp(filter2, insertOrUpdate2, false); + batchOperation = client.batchOperation(tableName); + batchOperation.addOperation(checkAndInsUp2); + result = batchOperation.execute(); + Assert.assertTrue(false); // cannot reach here + } catch (ObTableException e) { + e.printStackTrace(); + Assert + .assertEquals(ResultCodes.OB_KV_COLUMN_TYPE_NOT_MATCH.errorCode, e.getErrorCode()); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientAutoIncTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientAutoIncTest.java new file mode 100644 index 00000000..4b5bf225 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientAutoIncTest.java @@ -0,0 +1,567 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.bolt.ObTableClientTestBase; +import com.alipay.oceanbase.rpc.bolt.ObTableTest; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.filter.ObCompareOp; +import com.alipay.oceanbase.rpc.filter.ObTableValueFilter; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.property.Property; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.table.api.Table; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.*; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map; + +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.compareVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.row; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ObTableClientAutoIncTest { + public Table client; + + public void setClient(Table client) { + this.client = client; + } + + @After + public void close() throws Exception { + if (null != this.client && this.client instanceof ObTableClient) { + ((ObTableClient) this.client).close(); + } + } + + @Before + public void setup() throws Exception { + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.setMetadataRefreshInterval(100); + obTableClient.addProperty(Property.RPC_CONNECT_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.RPC_LOGIN_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.SERVER_CONNECTION_POOL_SIZE.getKey(), "1"); + obTableClient.addProperty(Property.RPC_EXECUTE_TIMEOUT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_MAX_WAIT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_EXECUTOR.getKey(), "32"); + obTableClient.addProperty(Property.RPC_OPERATION_TIMEOUT.getKey(), "3000"); + obTableClient.init(); + obTableClient.addRowKeyElement("test_auto_increment_rowkey", new String[] { "c1", "c2" }); + obTableClient.addRowKeyElement("test_auto_increment_not_rowkey", new String[] { "c1" }); + + this.client = obTableClient; + } + + private void executeSQL(String createSQL) throws SQLException { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute(createSQL); + } + + private void dropTable(String tableName) throws SQLException { + // use sql to drop table + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("drop table " + tableName); + } + + @Test + // Test auto increment on rowkey + public void testAutoIncrementRowkey() throws Exception { + // todo: only support in 4.x currently + if (ObTableClientTestUtil.isOBVersionLessThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + final String TABLE_NAME = "test_auto_increment_rowkey"; + + try { + executeSQL("CREATE TABLE IF NOT EXISTS `test_auto_increment_rowkey` (" + + "`c1` int auto_increment," + + "`c2` int NOT NULL," + + "`c3` int DEFAULT NULL," + + "`c4` varchar(255) DEFAULT NULL," + + "PRIMARY KEY(`c1`, `c2`)) partition by range columns(`c2`)" + + "(PARTITION p0 VALUES LESS THAN (100), PARTITION p1 VALUES LESS THAN (1000));"); + + client.insert(TABLE_NAME, new Object[] { 0, 1 }, new String[] { "c3" }, + new Object[] { 1 }); + + TableQuery tableQuery = client.query(TABLE_NAME); + tableQuery.select("c1", "c2", "c3"); + tableQuery.addScanRange(new Object[] { 1, 1 }, new Object[] { 200, 90 }); + ObTableValueFilter filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 1); + tableQuery.setFilter(filter); + QueryResultSet result = tableQuery.execute(); + Assert.assertTrue(result.next()); + Map value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(1, value.get("c3")); + + // test insert use user value + client.insert(TABLE_NAME, new Object[] { 100, 1 }, new String[] { "c3" }, + new Object[] { 1 }); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 100); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(1, value.get("c3")); + + // test insert sync global auto inc val + client.insert(TABLE_NAME, new Object[] { 0, 1 }, new String[] { "c3" }, + new Object[] { 1 }); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 101); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(1, value.get("c3")); + + // test delete + client.delete(TABLE_NAME, new Object[] { 101, 1 }); + + // test confirm delete + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 101); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertFalse(result.next()); + + // test update + ObTableValueFilter filter_3 = compareVal(ObCompareOp.EQ, "c3", 1); + + MutationResult updateResult = client.update(TABLE_NAME) + .setRowKey(colVal("c1", 1), colVal("c2", 1)).setFilter(filter_3) + .addMutateRow(row(colVal("c3", 5))).execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 1); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(5, value.get("c3")); + + // test replace not exist, insert + MutationResult theResult = client.replace(TABLE_NAME) + .setRowKey(colVal("c1", 0), colVal("c2", 1)).addMutateRow(row(colVal("c3", 2))) + .execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 102); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(2, value.get("c3")); + + // test replace exist, replace + theResult = client.replace(TABLE_NAME).setRowKey(colVal("c1", 101), colVal("c2", 1)) + .addMutateRow(row(colVal("c3", 20))).execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 101); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(20, value.get("c3")); + + // test insertup not exist, insert + theResult = client.insertOrUpdate(TABLE_NAME) + .setRowKey(colVal("c1", 0), colVal("c2", 1)).addMutateRow(row(colVal("c3", 5))) + .execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 103); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(5, value.get("c3")); + + // test insertup exist, update + theResult = client.insertOrUpdate(TABLE_NAME) + .setRowKey(colVal("c1", 103), colVal("c2", 1)).addMutateRow(row(colVal("c3", 50))) + .execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 103); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(50, value.get("c3")); + + // test insertup exist, update again + theResult = client.insertOrUpdate(TABLE_NAME) + .setRowKey(colVal("c1", 103), colVal("c2", 1)).addMutateRow(row(colVal("c3", 50))) + .execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 103); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(50, value.get("c3")); + + // test increment not exist, insert + value = client.increment(TABLE_NAME, new Object[] { 0, 1 }, new String[] { "c3" }, + new Object[] { 6 }, true); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 104); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(6, value.get("c3")); + + // test increment exist, increment + value = client.increment(TABLE_NAME, new Object[] { 104, 1 }, new String[] { "c3" }, + new Object[] { 6 }, true); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 104); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals(12, value.get("c3")); + + // test illegal increment on auto increment column + try { + value = client.increment(TABLE_NAME, new Object[] { 104, 1 }, + new String[] { "c1" }, new Object[] { 1 }, true); + } catch (ObTableException e) { + assertNotNull(e); + assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, e.getErrorCode()); + } + + // test append not exist, insert + Map res = client.append(TABLE_NAME, new Object[] { 0, 1 }, + new String[] { "c4" }, new Object[] { "a" }, true); + + tableQuery.select("c1", "c2", "c3", "c4"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 105); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals("a", value.get("c4")); + + // test append exist, append + res = client.append(TABLE_NAME, new Object[] { 105, 1 }, new String[] { "c4" }, + new Object[] { "b" }, true); + + tableQuery.select("c1", "c2", "c3", "c4"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 105); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals("ab", value.get("c4")); + + // the total number of data + tableQuery.select("c1", "c2", "c3", "c4"); + filter = new ObTableValueFilter(ObCompareOp.LT, "c1", 300); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertEquals(7, result.cacheSize()); + } finally { // drop table + dropTable(TABLE_NAME); + } + } + + @Test + // Test auto increment on not rowkey + public void testAutoIncrementNotRowkey() throws Exception { + // todo: only support in 4.x currently + if (ObTableClientTestUtil.isOBVersionLessThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + final String TABLE_NAME = "test_auto_increment_not_rowkey"; + + try { + executeSQL("CREATE TABLE IF NOT EXISTS `test_auto_increment_not_rowkey` (" + + "`c1` int NOT NULL," + + "`c2` int DEFAULT NULL," + + "`c3` tinyint auto_increment," + + "`c4` varchar(255) DEFAULT NULL," + + "PRIMARY KEY(`c1`)) partition by range columns(`c1`)" + + "(PARTITION p0 VALUES LESS THAN (100), PARTITION p1 VALUES LESS THAN (1000));"); + + client + .insert(TABLE_NAME, new Object[] { 1 }, new String[] { "c2" }, new Object[] { 1 }); + + TableQuery tableQuery = client.query(TABLE_NAME); + tableQuery.select("c1", "c2", "c3"); + ObTableValueFilter filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 1); + tableQuery.setFilter(filter); + tableQuery.addScanRange(new Object[] { 0 }, new Object[] { 90 }); + QueryResultSet result = tableQuery.execute(); + Assert.assertTrue(result.next()); + Map value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals((byte) 1, value.get("c3")); + + // test insert use user value + client.insert(TABLE_NAME, new Object[] { 2 }, new String[] { "c2", "c3" }, + new Object[] { 1, (byte) 100 }); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 2); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals((byte) 100, value.get("c3")); + + // test insert sync global auto inc val + client + .insert(TABLE_NAME, new Object[] { 3 }, new String[] { "c2" }, new Object[] { 1 }); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 3); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals((byte) 101, value.get("c3")); + + // test delete + client.delete(TABLE_NAME, new Object[] { 1 }); + + // test confirm delete + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 1); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertFalse(result.next()); + + // test update + ObTableValueFilter filter_3 = compareVal(ObCompareOp.EQ, "c2", 1); + + MutationResult updateResult = client.update(TABLE_NAME).setRowKey(colVal("c1", 3)) + .setFilter(filter_3).addMutateRow(row(colVal("c3", (byte) 5))).execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 3); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals((byte) 5, value.get("c3")); + + // test replace not exist, insert + MutationResult theResult = client.replace(TABLE_NAME).setRowKey(colVal("c1", 4)) + .addMutateRow(row(colVal("c2", 1))).execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 4); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals((byte) 102, value.get("c3")); + + // test replace exist, replace + theResult = client.replace(TABLE_NAME).setRowKey(colVal("c1", 3)) + .addMutateRow(row(colVal("c3", (byte) 20), colVal("c2", 1))).execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 3); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals((byte) 20, value.get("c3")); + + // test insertup not exist, insert + theResult = client.insertOrUpdate(TABLE_NAME).setRowKey(colVal("c1", 5)) + .addMutateRow(row(colVal("c2", 1))).execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 5); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals((byte) 103, value.get("c3")); + + // test insertup exist, update + theResult = client.insertOrUpdate(TABLE_NAME).setRowKey(colVal("c1", 5)) + .addMutateRow(row(colVal("c3", (byte) 50), colVal("c2", 1))).execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 5); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals((byte) 50, value.get("c3")); + + // test insertup exist, update again + theResult = client.insertOrUpdate(TABLE_NAME).setRowKey(colVal("c1", 5)) + .addMutateRow(row(colVal("c3", (byte) 50), colVal("c2", 1))).execute(); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 5); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(1, value.get("c2")); + Assert.assertEquals((byte) 50, value.get("c3")); + + // test increment not exist, insert + value = client.increment(TABLE_NAME, new Object[] { 6 }, new String[] { "c2" }, + new Object[] { 6 }, true); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 6); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(6, value.get("c2")); + Assert.assertEquals((byte) 104, value.get("c3")); + + // test increment exist, increment + value = client.increment(TABLE_NAME, new Object[] { 6 }, new String[] { "c2" }, + new Object[] { 6 }, true); + + tableQuery.select("c1", "c2", "c3"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 6); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals(12, value.get("c2")); + Assert.assertEquals((byte) 104, value.get("c3")); + + // test illegal increment on auto increment column + try { + value = client.increment(TABLE_NAME, new Object[] { 6 }, new String[] { "c3" }, + new Object[] { (byte) 5 }, true); + } catch (ObTableException e) { + assertNotNull(e); + assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, e.getErrorCode()); + } + + // test append not exist, insert + Map res = client.append(TABLE_NAME, new Object[] { 200 }, + new String[] { "c4" }, new Object[] { "a" }, true); + + tableQuery.select("c1", "c2", "c3", "c4"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 200); + tableQuery.setFilter(filter); + tableQuery.addScanRange(new Object[] { 100 }, new Object[] { 200 }); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals((byte) 105, value.get("c3")); + Assert.assertEquals("a", value.get("c4")); + + // test append exist, append + res = client.append(TABLE_NAME, new Object[] { 200 }, new String[] { "c4" }, + new Object[] { "b" }, true); + + tableQuery.select("c1", "c2", "c3", "c4"); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 200); + tableQuery.setFilter(filter); + tableQuery.addScanRange(new Object[] { 100 }, new Object[] { 200 }); + result = tableQuery.execute(); + Assert.assertTrue(result.next()); + value = result.getRow(); + Assert.assertEquals((byte) 105, value.get("c3")); + Assert.assertEquals("ab", value.get("c4")); + + // the total number of data + tableQuery = client.query(TABLE_NAME); + tableQuery.addScanRange(new Object[] { 0 }, new Object[] { 90 }); + filter = new ObTableValueFilter(ObCompareOp.LT, "c1", 90); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertEquals(5, result.cacheSize()); + + // the total number of data + tableQuery = client.query(TABLE_NAME); + tableQuery.addScanRange(new Object[] { 100 }, new Object[] { 300 }); + filter = new ObTableValueFilter(ObCompareOp.EQ, "c1", 200); + tableQuery.setFilter(filter); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + } finally { // drop table + dropTable(TABLE_NAME); + } + } + + // CREATE TABLE IF NOT EXISTS `test_auto_increment_one_rowkey` (`c1` int auto_increment, `c2` int NOT NULL, PRIMARY KEY(`c1`)); + @Test + // test insert null into auto increment column + public void testAutoColumnRowKey() throws Exception { + final String TABLE_NAME = "test_auto_increment_one_rowkey"; + try { + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement(TABLE_NAME, new String[] { "c1" }); + try { + client.insert(TABLE_NAME, null, new String[] { "c2" }, new Object[] { 1 }); + } catch (Exception e) { + Assert.assertEquals("Cannot read the array length because \"rowKeys\" is null", + ((NullPointerException) e).getMessage()); + } + } finally { + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientCheckAndInsertTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientCheckAndInsertTest.java new file mode 100644 index 00000000..66885bf6 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientCheckAndInsertTest.java @@ -0,0 +1,178 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.bolt.ObTableClientTestBase; +import com.alipay.oceanbase.rpc.bolt.ObTableTest; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.filter.ObCompareOp; +import com.alipay.oceanbase.rpc.filter.ObTableValueFilter; +import com.alipay.oceanbase.rpc.location.model.ObServerAddr; +import com.alipay.oceanbase.rpc.location.model.ServerRoster; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.property.Property; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.table.api.Table; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.*; + +import java.lang.reflect.Field; + +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.compareVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.row; + +public class ObTableClientCheckAndInsertTest { + + public Table client; + + public void setClient(Table client) { + this.client = client; + } + + @After + public void close() throws Exception { + if (null != this.client && this.client instanceof ObTableClient) { + ((ObTableClient) this.client).close(); + } + } + + @Before + public void setup() throws Exception { + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.setMetadataRefreshInterval(100); + obTableClient.addProperty(Property.RPC_CONNECT_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.RPC_LOGIN_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.SERVER_CONNECTION_POOL_SIZE.getKey(), "1"); + obTableClient.addProperty(Property.RPC_EXECUTE_TIMEOUT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_MAX_WAIT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_EXECUTOR.getKey(), "32"); + obTableClient.addProperty(Property.RPC_OPERATION_TIMEOUT.getKey(), "3000"); + obTableClient.init(); + + this.client = obTableClient; + } + + @Test + // test check and insert + public void testCheckAndInsert() throws Exception { + // todo: only support in 4.x currently + if (ObTableClientTestUtil.isOBVersionLessThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + final String TABLE_NAME = "test_mutation"; + + TableQuery tableQuery = client.query(TABLE_NAME); + tableQuery.addScanRange(new Object[] { 0L, "\0" }, new Object[] { 200L, "\254" }); + tableQuery.select("c1", "c2", "c3", "c4"); + + try { + // prepare data with insert + client.insert(TABLE_NAME).setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 100L)).execute(); + client.insert(TABLE_NAME).setRowKey(colVal("c1", 1L), colVal("c2", "row_1")) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 101L)).execute(); + client.insert(TABLE_NAME).setRowKey(colVal("c1", 2L), colVal("c2", "row_2")) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 102L)).execute(); + client.insert(TABLE_NAME).setRowKey(colVal("c1", 3L), colVal("c2", "row_3")) + .addMutateColVal(colVal("c1", 3L)).addMutateColVal(colVal("c2", "row_3")) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 103L)).execute(); + + // insert / match filter + ObTableValueFilter c4_EQ_101 = compareVal(ObCompareOp.EQ, "c4", 101L); + MutationResult insertResult = client.insert(TABLE_NAME) + .setRowKey(colVal("c1", 100L), colVal("c2", "row_5")).setFilter(c4_EQ_101) + .addScanRange(new Object[] { 0L, "\0" }, new Object[] { 200L, "\254" }) + .addMutateRow(row(colVal("c3", new byte[] { 1 }), colVal("c4", 999L))).execute(); + Assert.assertEquals(1, insertResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_0 = compareVal(ObCompareOp.EQ, "c4", 999L); + tableQuery.setFilter(confirm_0); + QueryResultSet result_0 = tableQuery.execute(); + Assert.assertEquals(1, result_0.cacheSize()); + + insertResult = client.insert(TABLE_NAME) + .setRowKey(colVal("c1", 120L), colVal("c2", "row_6")).setFilter(c4_EQ_101) + .addScanRange(new Object[] { 0L, "\0" }, new Object[] { 200L, "\254" }) + .addMutateRow(row(colVal("c3", new byte[] { 1 }), colVal("c4", 999L))).execute(); + Assert.assertEquals(1, insertResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + confirm_0 = compareVal(ObCompareOp.EQ, "c4", 999L); + tableQuery.setFilter(confirm_0); + result_0 = tableQuery.execute(); + Assert.assertEquals(2, result_0.cacheSize()); + + // insert / only insert one row when multiple match + ObTableValueFilter c4_EQ_999 = compareVal(ObCompareOp.EQ, "c4", 999L); + insertResult = client.insert(TABLE_NAME) + .setRowKey(colVal("c1", 130L), colVal("c2", "row_7")).setFilter(c4_EQ_999) + .addScanRange(new Object[] { 0L, "\0" }, new Object[] { 200L, "\254" }) + .addMutateRow(row(colVal("c3", new byte[] { 1 }), colVal("c4", 99L))).execute(); + Assert.assertEquals(1, insertResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + confirm_0 = compareVal(ObCompareOp.EQ, "c4", 99L); + tableQuery.setFilter(confirm_0); + result_0 = tableQuery.execute(); + Assert.assertEquals(1, result_0.cacheSize()); + + // insert / do not match filter + ObTableValueFilter c4_EQ_201 = compareVal(ObCompareOp.EQ, "c4", 201L); + insertResult = client.insert(TABLE_NAME) + .setRowKey(colVal("c1", 150L), colVal("c2", "row_8")).setFilter(c4_EQ_201) + .addScanRange(new Object[] { 0L, "\0" }, new Object[] { 200L, "\254" }) + .addMutateRow(row(colVal("c3", new byte[] { 1 }), colVal("c4", 4000L))).execute(); + Assert.assertEquals(0, insertResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + confirm_0 = compareVal(ObCompareOp.EQ, "c4", 4000L); + tableQuery.setFilter(confirm_0); + result_0 = tableQuery.execute(); + Assert.assertEquals(0, result_0.cacheSize()); + + // test defense for set scan range without filter + try { + insertResult = client.insert(TABLE_NAME) + .setRowKey(colVal("c1", 120L), colVal("c2", "row_7")) + .addScanRange(new Object[] { 0L, "\0" }, new Object[] { 200L, "\254" }) + .addMutateRow(row(colVal("c3", new byte[] { 1 }), colVal("c4", 999L))) + .execute(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableUnexpectedException); + Assert.assertEquals("should set filter and scan range both", e.getMessage()); + } + } finally { + client.delete(TABLE_NAME).setRowKey(colVal("c1", 0L), colVal("c2", "row_0")).execute(); + client.delete(TABLE_NAME).setRowKey(colVal("c1", 1L), colVal("c2", "row_1")).execute(); + client.delete(TABLE_NAME).setRowKey(colVal("c1", 2L), colVal("c2", "row_2")).execute(); + client.delete(TABLE_NAME).setRowKey(colVal("c1", 3L), colVal("c2", "row_3")).execute(); + client.delete(TABLE_NAME).setRowKey(colVal("c1", 100L), colVal("c2", "row_5")) + .execute(); + client.delete(TABLE_NAME).setRowKey(colVal("c1", 120L), colVal("c2", "row_6")) + .execute(); + client.delete(TABLE_NAME).setRowKey(colVal("c1", 130L), colVal("c2", "row_7")) + .execute(); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientITCase.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientITCase.java new file mode 100644 index 00000000..8003b6e9 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientITCase.java @@ -0,0 +1,68 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.containerBase.ContainerTestBase; +import com.alipay.oceanbase.rpc.property.Property; +import com.alipay.oceanbase.rpc.table.api.Table; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ObTableClientITCase extends ContainerTestBase { + + private static final Logger logger = LoggerFactory.getLogger(ObTableClientITCase.class); + + public Table client; + + @Test + public void testAll() throws Exception { + if (!ObTableClientTestUtil.FULL_USER_NAME.equals(TEST_USERNAME)) { + return; + } + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + + obTableClient.setMetadataRefreshInterval(100); + obTableClient.addProperty(Property.RPC_CONNECT_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.RPC_LOGIN_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.SERVER_CONNECTION_POOL_SIZE.getKey(), "1"); + obTableClient.addProperty(Property.RPC_EXECUTE_TIMEOUT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_MAX_WAIT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_EXECUTOR.getKey(), "32"); + obTableClient.addProperty(Property.RPC_OPERATION_TIMEOUT.getKey(), "3000"); + obTableClient.addProperty(Property.SERVER_ENABLE_REROUTING.getKey(), "False"); + obTableClient.init(); + + logger.info("obTableClient init success"); + + // ObTableClientTest + ObTableClientTest obTableClientTest = new ObTableClientTest(); + obTableClientTest.setClient(obTableClient); + obTableClientTest.test_batch(); + obTableClientTest.testBatchMutation(); + obTableClientTest.testMultiThreadBatchOperation(); + obTableClientTest.testCompareWithNull(); + obTableClientTest.testQueryFilterLimit(); + // Todo: add more test + logger.info("ObTableClientTest success"); + + logger.info("testAll success"); + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientInitTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientInitTest.java index 5fb1aa90..cbad14a9 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientInitTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientInitTest.java @@ -36,7 +36,6 @@ public void testInit() throws Exception { // builder = new ObTable.Builder("1,1,1,1", 123); // PowerMockito.whenNew(ObTable.Builder.class).withAnyArguments().thenReturn(builder); client = ObTableClientTestUtil.newTestClient(); - ; client.init(); } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionHashTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionHashTest.java index 492ba766..d265e7d8 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionHashTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionHashTest.java @@ -17,17 +17,21 @@ package com.alipay.oceanbase.rpc; +import com.alipay.oceanbase.rpc.property.Property; import com.alipay.oceanbase.rpc.stream.QueryResultSet; import com.alipay.oceanbase.rpc.table.api.TableBatchOps; import com.alipay.oceanbase.rpc.table.api.TableQuery; import com.alipay.oceanbase.rpc.threadlocal.ThreadLocalMap; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import net.bytebuddy.implementation.auxiliary.MethodCallProxy; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.util.List; -import java.util.Map; +import java.sql.Connection; +import java.sql.Statement; +import java.util.*; import java.util.concurrent.Executors; public class ObTableClientPartitionHashTest { @@ -49,12 +53,20 @@ public void setup() throws Exception { obTableClient.setTableEntryAcquireSocketTimeout(10000); obTableClient.addProperty("connectTimeout", "100000"); obTableClient.addProperty("socketTimeout", "100000"); + obTableClient.addProperty(Property.RPC_EXECUTE_TIMEOUT.getKey(), "5000"); obTableClient.setRunningMode(ObTableClient.RunningMode.HBASE); obTableClient.init(); this.obTableClient = obTableClient; } + @After + public void close() throws Exception { + if (null != this.obTableClient) { + ((ObTableClient) this.obTableClient).close(); + } + } + @Test public void testInsert() throws Exception { long timestamp = System.currentTimeMillis(); @@ -141,41 +153,404 @@ public void testDelete() throws Exception { @Test public void testQuery() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testHash", new Object[] { timeStamp, "partition".getBytes(), - timeStamp }, new String[] { "V" }, new Object[] { "value1L".getBytes() }); - obTableClient.insert("testHash", new Object[] { timeStamp + 5, "partition".getBytes(), - timeStamp }, new String[] { "V" }, new Object[] { "value2".getBytes() }); - - TableQuery tableQuery = obTableClient.query("testHash"); - tableQuery.addScanRange(new Object[] { timeStamp, "partition".getBytes(), timeStamp }, - new Object[] { timeStamp, "partition".getBytes(), timeStamp }); - tableQuery.select("K", "Q", "T", "V"); - QueryResultSet result = tableQuery.execute(); - Assert.assertEquals(1L, result.cacheSize()); - - tableQuery = obTableClient.query("testHash"); - tableQuery.addScanRange(new Object[] { timeStamp, "partition".getBytes(), timeStamp }, - new Object[] { timeStamp + 10, "partition".getBytes(), timeStamp }); - tableQuery.select("K", "Q", "T", "V"); - - result = tableQuery.execute(); - Assert.assertEquals(2, result.cacheSize()); - - tableQuery = obTableClient.query("testHash"); - tableQuery.limit(1); - tableQuery.addScanRange(new Object[] { timeStamp, "partition".getBytes(), timeStamp }, - new Object[] { timeStamp + 10, "partition".getBytes(), timeStamp }); - tableQuery.select("K", "Q", "T", "V"); - result = tableQuery.execute(); - // FIXME the limit is not work ? - Assert.assertEquals(2, result.cacheSize()); - - // server not supported - // tableQuery = obTableClient.query("testHash"); - // tableQuery.addScanRange(new Object[]{timeStamp + "", "partition".getBytes(), timeStamp}, new Object[]{timeStamp + 1L0 + "", "partition".getBytes(), timeStamp}); - // tableQuery.select("K", "Q", "T", "V"); - // result = tableQuery.execute(); - // Assert.assertEquals(2, result.size()); + try { + obTableClient.insert("testHash", new Object[] { timeStamp, "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value0".getBytes() }); + obTableClient.insert("testHash", new Object[] { timeStamp + 1, "partition".getBytes(), + timeStamp + 1 }, new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testHash", new Object[] { timeStamp + 2, "partition".getBytes(), + timeStamp + 2 }, new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert("testHash", new Object[] { timeStamp + 3, "partition".getBytes(), + timeStamp + 3 }, new String[] { "V" }, new Object[] { "value3".getBytes() }); + obTableClient.insert("testHash", new Object[] { timeStamp + 4, "partition".getBytes(), + timeStamp + 4 }, new String[] { "V" }, new Object[] { "value4".getBytes() }); + + // query with one partition + TableQuery tableQuery = obTableClient.query("testHash"); + tableQuery.addScanRange(new Object[] { timeStamp, "partition".getBytes(), timeStamp }, + new Object[] { timeStamp, "partition".getBytes(), timeStamp }); + tableQuery.select("Q", "T", "K", "V"); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(1L, result.cacheSize()); + + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals(timeStamp, row.get("K")); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value0", new String((byte[]) row.get("V"))); + Assert.assertFalse(result.next()); + + // query with one partition using prefix + tableQuery = obTableClient.query("testHash"); + tableQuery.addScanRange(new Object[] { timeStamp }, new Object[] { timeStamp }); + tableQuery.setScanRangeColumns("K"); + tableQuery.select("K", "Q", "T", "V"); + result = tableQuery.execute(); + Assert.assertEquals(1L, result.cacheSize()); + + Assert.assertTrue(result.next()); + row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals(timeStamp, row.get("K")); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value0", new String((byte[]) row.get("V"))); + Assert.assertFalse(result.next()); + + // query with multiply partitions + tableQuery = obTableClient.query("testHash"); + tableQuery.addScanRange( + new Object[] { timeStamp, "partition".getBytes(), timeStamp }, new Object[] { + timeStamp + 10, "partition".getBytes(), timeStamp }); + tableQuery.select("K", "Q", "T"); + result = tableQuery.execute(); + Assert.assertEquals(5, result.cacheSize()); + + // query with multiply partitions using prefix K + tableQuery = obTableClient.query("testHash"); + tableQuery.setScanRangeColumns("K"); + tableQuery + .addScanRange(new Object[] { timeStamp }, new Object[] { timeStamp + 10 }); + tableQuery.select("Q", "V"); + result = tableQuery.execute(); + Assert.assertEquals(5, result.cacheSize()); + + // query with empty scan range + tableQuery = obTableClient.query("testHash"); + result = tableQuery.execute(); + Assert.assertEquals(5, result.cacheSize()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testHash", new Object[] { timeStamp, "partition".getBytes(), + timeStamp }); + obTableClient.delete("testHash", new Object[] { timeStamp + 1, "partition".getBytes(), + timeStamp + 1 }); + obTableClient.delete("testHash", new Object[] { timeStamp + 2, "partition".getBytes(), + timeStamp + 2 }); + obTableClient.delete("testHash", new Object[] { timeStamp + 3, "partition".getBytes(), + timeStamp + 3 }); + obTableClient.delete("testHash", new Object[] { timeStamp + 4, "partition".getBytes(), + timeStamp + 4 }); + } + } + + @Test + public void testAsyncQuery() throws Exception { + long timeStamp = System.currentTimeMillis(); + try { + obTableClient.insert("testHash", new Object[] { timeStamp, "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value0".getBytes() }); + obTableClient.insert("testHash", new Object[] { timeStamp + 1, "partition".getBytes(), + timeStamp + 1 }, new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testHash", new Object[] { timeStamp + 2, "partition".getBytes(), + timeStamp + 2 }, new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert("testHash", new Object[] { timeStamp + 3, "partition".getBytes(), + timeStamp + 3 }, new String[] { "V" }, new Object[] { "value3".getBytes() }); + obTableClient.insert("testHash", new Object[] { timeStamp + 4, "partition".getBytes(), + timeStamp + 4 }, new String[] { "V" }, new Object[] { "value4".getBytes() }); + + // query with one partition + TableQuery tableQuery = obTableClient.query("testHash"); + tableQuery.addScanRange(new Object[] { timeStamp, "partition".getBytes(), timeStamp }, + new Object[] { timeStamp, "partition".getBytes(), timeStamp }); + tableQuery.select("Q", "T", "K", "V"); + tableQuery.setBatchSize(1); + QueryResultSet result = tableQuery.asyncExecute(); + Assert.assertEquals(1L, result.cacheSize()); + + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals(timeStamp, row.get("K")); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value0", new String((byte[]) row.get("V"))); + Assert.assertFalse(result.next()); + + // query with one partition using prefix + tableQuery = obTableClient.query("testHash"); + tableQuery.addScanRange(new Object[] { timeStamp }, new Object[] { timeStamp }); + tableQuery.setScanRangeColumns("K"); + tableQuery.select("K", "Q", "T", "V"); + tableQuery.setBatchSize(1); + result = tableQuery.asyncExecute(); + Assert.assertEquals(1L, result.cacheSize()); + + Assert.assertTrue(result.next()); + row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals(timeStamp, row.get("K")); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value0", new String((byte[]) row.get("V"))); + Assert.assertFalse(result.next()); + + // query with multiply partitions + tableQuery = obTableClient.query("testHash"); + tableQuery.addScanRange( + new Object[] { timeStamp, "partition".getBytes(), timeStamp }, new Object[] { + timeStamp + 10, "partition".getBytes(), timeStamp }); + tableQuery.select("K", "Q", "T"); + tableQuery.setBatchSize(2); + result = tableQuery.asyncExecute(); + + // query with multiply partitions using prefix K + tableQuery = obTableClient.query("testHash"); + tableQuery.setScanRangeColumns("K"); + tableQuery + .addScanRange(new Object[] { timeStamp }, new Object[] { timeStamp + 10 }); + tableQuery.select("Q", "V"); + tableQuery.setBatchSize(1); + result = tableQuery.asyncExecute(); + + // query with empty scan range + tableQuery = obTableClient.query("testHash"); + result = tableQuery.execute(); + Assert.assertEquals(5, result.cacheSize()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testHash", new Object[] { timeStamp, "partition".getBytes(), + timeStamp }); + obTableClient.delete("testHash", new Object[] { timeStamp + 1, "partition".getBytes(), + timeStamp + 1 }); + obTableClient.delete("testHash", new Object[] { timeStamp + 2, "partition".getBytes(), + timeStamp + 2 }); + obTableClient.delete("testHash", new Object[] { timeStamp + 3, "partition".getBytes(), + timeStamp + 3 }); + obTableClient.delete("testHash", new Object[] { timeStamp + 4, "partition".getBytes(), + timeStamp + 4 }); + } + } + + @Test + public void testQueryLocalIndex() throws Exception { + long timeStamp = System.currentTimeMillis(); + String tableName = "testHash"; + try { + obTableClient.insert("testHash", + new Object[] { timeStamp + 1, "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert("testHash", + new Object[] { timeStamp + 1, "partition".getBytes(), timeStamp + 2 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testHash", + new Object[] { timeStamp + 2, "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testHash", + new Object[] { timeStamp + 2, "partition".getBytes(), timeStamp + 2 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + + // query timestamp + 1 + TableQuery tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { timeStamp + 1, "value0".getBytes() }, + new Object[] { timeStamp + 1, "value9".getBytes() }); + tableQuery.scanOrder(false); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals(timeStamp + 1, row.get("K")); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + ( 3 - i), new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // query timestamp + 1 using prefix + tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { timeStamp + 1 }, + new Object[] { timeStamp + 1 }); + tableQuery.select("K", "T", "V"); + tableQuery.setScanRangeColumns("K"); + tableQuery.scanOrder(false); + tableQuery.indexName("i1"); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(3, row.size()); + Assert.assertEquals(timeStamp + 1, row.get("K")); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + ( 3 - i), new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // query with mutliply partition + tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { timeStamp + 1, "value0".getBytes() }, + new Object[] { timeStamp + 3, "value9".getBytes() }); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + result = tableQuery.execute(); + Assert.assertEquals(4, result.cacheSize()); + + // sort result by K, T + List> resultList = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + resultList.add(result.getRow()); + } + Assert.assertFalse(result.next()); + Collections.sort(resultList, new Comparator>() { + @Override + public int compare(Map o1, Map o2) { + int firstCmp = (int)(((long)o1.get("K") - (long)o2.get("K"))); + if (firstCmp == 0) { + return (int)(((long)o1.get("T")) - ((long)o2.get("T"))); + } + return firstCmp; + } + }); + int[] orderedDeltaKeys = {1, 1, 2, 2}; + int[] orderedDeltaTs = {1, 2, 1, 2}; + String[] orderedValues = {"value2", "value1", "value1", "value2"}; + for (int i = 0; i < 4; i++) { + Assert.assertEquals(timeStamp + orderedDeltaKeys[i], resultList.get(i).get("K")); + Assert.assertEquals("partition", new String((byte[]) resultList.get(i).get("Q"))); + Assert.assertEquals(timeStamp + orderedDeltaTs[i], resultList.get(i).get("T")); + Assert.assertEquals(orderedValues[i], new String((byte[]) resultList.get(i).get("V"))); + } + Assert.assertFalse(result.next()); + + } catch (Exception e){ + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testHash", + new Object[] { timeStamp + 1, "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete("testHash", + new Object[] { timeStamp + 1, "partition".getBytes(), timeStamp + 2 }); + obTableClient.delete("testHash", + new Object[] { timeStamp + 2, "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete("testHash", + new Object[] { timeStamp + 2, "partition".getBytes(), timeStamp + 2 }); + } + } + + @Test + public void testAsyncQueryLocalIndex() throws Exception { + long timeStamp = System.currentTimeMillis(); + String tableName = "testHash"; + try { + obTableClient.insert("testHash", + new Object[] { timeStamp + 1, "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert("testHash", + new Object[] { timeStamp + 1, "partition".getBytes(), timeStamp + 2 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testHash", + new Object[] { timeStamp + 2, "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testHash", + new Object[] { timeStamp + 2, "partition".getBytes(), timeStamp + 2 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + + // query timestamp + 1 + TableQuery tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { timeStamp + 1, "value0".getBytes() }, + new Object[] { timeStamp + 1, "value9".getBytes() }); + tableQuery.scanOrder(false); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(1); + QueryResultSet result = tableQuery.asyncExecute(); + Assert.assertEquals(1, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals(timeStamp + 1, row.get("K")); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + ( 3 - i), new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // query timestamp + 1 using prefix + tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { timeStamp + 1 }, + new Object[] { timeStamp + 1 }); + tableQuery.select("K", "T", "V"); + tableQuery.setScanRangeColumns("K"); + tableQuery.scanOrder(false); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(1); + result = tableQuery.asyncExecute(); + Assert.assertEquals(1, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(3, row.size()); + Assert.assertEquals(timeStamp + 1, row.get("K")); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + ( 3 - i), new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // query with mutliply partition + tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { timeStamp + 1, "value0".getBytes() }, + new Object[] { timeStamp + 3, "value9".getBytes() }); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(2); + result = tableQuery.asyncExecute(); + Assert.assertEquals(2, result.cacheSize()); + + // sort result by K, T + List> resultList = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + resultList.add(result.getRow()); + } + Assert.assertFalse(result.next()); + Collections.sort(resultList, new Comparator>() { + @Override + public int compare(Map o1, Map o2) { + int firstCmp = (int)(((long)o1.get("K") - (long)o2.get("K"))); + if (firstCmp == 0) { + return (int)(((long)o1.get("T")) - ((long)o2.get("T"))); + } + return firstCmp; + } + }); + int[] orderedDeltaKeys = {1, 1, 2, 2}; + int[] orderedDeltaTs = {1, 2, 1, 2}; + String[] orderedValues = {"value2", "value1", "value1", "value2"}; + for (int i = 0; i < 4; i++) { + Assert.assertEquals(timeStamp + orderedDeltaKeys[i], resultList.get(i).get("K")); + Assert.assertEquals("partition", new String((byte[]) resultList.get(i).get("Q"))); + Assert.assertEquals(timeStamp + orderedDeltaTs[i], resultList.get(i).get("T")); + Assert.assertEquals(orderedValues[i], new String((byte[]) resultList.get(i).get("V"))); + } + Assert.assertFalse(result.next()); + + } catch (Exception e){ + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testHash", + new Object[] { timeStamp + 1, "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete("testHash", + new Object[] { timeStamp + 1, "partition".getBytes(), timeStamp + 2 }); + obTableClient.delete("testHash", + new Object[] { timeStamp + 2, "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete("testHash", + new Object[] { timeStamp + 2, "partition".getBytes(), timeStamp + 2 }); + } } @Test @@ -325,4 +700,40 @@ public void testBatchConcurrentWithPriority() throws Exception { Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); } + public void cleanPartitionLocationTable(String tableName) throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("delete from " + tableName); + } + + @Test + public void testPartitionLocation() throws Exception { + obTableClient.setRunningMode(ObTableClient.RunningMode.NORMAL); + String testTable = "testPartitionHashComplex"; + obTableClient.addRowKeyElement(testTable, new String[] { "c1", "c2" }); + Random rng = new Random(); + try { + cleanPartitionLocationTable(testTable); + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + for (int i = 0; i < 64; i++) { + int c1 = rng.nextInt(); + long c2 = rng.nextLong(); + + // use sql to insert data + statement.execute("insert into " + testTable + "(c1, c2, c3) values (" + c1 + "," + + c2 + "," + "'value')"); + + // get data by obkv interface + Map result = obTableClient.get(testTable, new Object[] { c1, c2 }, + new String[] { "c1", "c2", "c3" }); + Assert.assertEquals(3, result.size()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanPartitionLocationTable(testTable); + } + } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionKeyTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionKeyTest.java index 9794c413..dfee15d2 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionKeyTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionKeyTest.java @@ -17,24 +17,42 @@ package com.alipay.oceanbase.rpc; -import com.alipay.oceanbase.rpc.exception.ObTablePartitionConsistentException; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.filter.ObCompareOp; +import com.alipay.oceanbase.rpc.mutation.BatchOperation; +import com.alipay.oceanbase.rpc.mutation.Row; +import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.stream.QueryResultSet; import com.alipay.oceanbase.rpc.table.api.TableBatchOps; import com.alipay.oceanbase.rpc.table.api.TableQuery; import com.alipay.oceanbase.rpc.threadlocal.ThreadLocalMap; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.sql.Connection; +import java.sql.Statement; import java.util.List; import java.util.Map; import java.util.concurrent.Executors; +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.andList; +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.compareVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.*; +import static com.alipay.oceanbase.rpc.util.ObTableClientTestUtil.cleanTable; +import static com.alipay.oceanbase.rpc.util.ObTableClientTestUtil.generateRandomStringByUUID; +import static org.junit.Assert.fail; + public class ObTableClientPartitionKeyTest { private ObTableClient obTableClient; + private ObTableClient normalClient; + private String TEST_TABLE = null; @Before public void setUp() throws Exception { @@ -60,296 +78,1229 @@ primary key(K, Q, T) obTableClient.setRunningMode(ObTableClient.RunningMode.HBASE); obTableClient.init(); + final ObTableClient normalClient = ObTableClientTestUtil.newTestClient(); + normalClient.setMetadataRefreshInterval(100); + normalClient.addProperty("connectTimeout", "100000"); + normalClient.addProperty("socketTimeout", "100000"); + normalClient.setTableEntryAcquireSocketTimeout(10000); + normalClient.setRunningMode(ObTableClient.RunningMode.NORMAL); + normalClient.init(); + + if (obTableClient.isOdpMode()) { + TEST_TABLE = "testKey"; + } else { + TEST_TABLE = "testPartition"; + obTableClient.addRowKeyElement("testPartition", new String[] { "K", "Q", "T" }); + } + this.obTableClient = obTableClient; + this.normalClient = normalClient; + } + @After + public void close() throws Exception { + if (null != this.obTableClient) { + ((ObTableClient) this.obTableClient).close(); + } } @Test public void testInsert() throws Exception { long timestamp = System.currentTimeMillis(); - long affectRow = obTableClient.insert("testPartition", new Object[] { "partitionKey", - "partition".getBytes(), timestamp }, new String[] { "V" }, - new Object[] { "aa".getBytes() }); - Assert.assertEquals(1, affectRow); - - affectRow = obTableClient.insertOrUpdate("testPartition", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timestamp }, - new String[] { "V" }, new Object[] { "bb".getBytes() }); - Assert.assertEquals(1, affectRow); - - Map result = obTableClient.get("testPartition", new Object[] { - "partitionKey".getBytes(), "partition".getBytes(), timestamp }, new String[] { "K", - "Q", "T", "V", "K_PREFIX" }); - Assert.assertEquals("partitionKey", new String((byte[]) result.get("K"), "UTF-8")); - Assert.assertEquals("partition", new String((byte[]) result.get("Q"), "UTF-8")); - Assert.assertEquals(timestamp, result.get("T")); - Assert.assertEquals("bb", new String((byte[]) result.get("V"), "UTF-8")); + try { + long affectRow = obTableClient.insert(TEST_TABLE, new Object[] { "partitionKey", + "partition".getBytes(), timestamp }, new String[] { "V" }, + new Object[] { "aa".getBytes() }); + Assert.assertEquals(1, affectRow); + + affectRow = obTableClient.insertOrUpdate(TEST_TABLE, + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timestamp }, + new String[] { "V" }, new Object[] { "bb".getBytes() }); + Assert.assertEquals(1, affectRow); + Map result = obTableClient.get(TEST_TABLE, new Object[] { + "partitionKey".getBytes(), "partition".getBytes(), timestamp }, new String[] { + "K", "Q", "T", "V" }); + Assert.assertEquals("partitionKey", new String((byte[]) result.get("K"), "UTF-8")); + Assert.assertEquals("partition", new String((byte[]) result.get("Q"), "UTF-8")); + Assert.assertEquals(timestamp, result.get("T")); + Assert.assertEquals("bb", new String((byte[]) result.get("V"), "UTF-8")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete(TEST_TABLE, new Object[] { "partitionKey", "partition".getBytes(), + timestamp }); + } } @Test public void testGet() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testPartition", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value".getBytes() }); - Map result = obTableClient.get("testPartition", new Object[] { - "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { "K", - "Q", "T", "V", "K_PREFIX" }); - Assert.assertEquals("partitionKey", new String((byte[]) result.get("K"), "UTF-8")); - Assert.assertEquals("partition", new String((byte[]) result.get("Q"), "UTF-8")); - Assert.assertEquals(timeStamp, result.get("T")); - Assert.assertEquals("value", new String((byte[]) result.get("V"), "UTF-8")); + try { + obTableClient.insert(TEST_TABLE, + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value".getBytes() }); + Map result = obTableClient.get(TEST_TABLE, new Object[] { + "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { + "K", "Q", "T", "V" }); + Assert.assertEquals("partitionKey", new String((byte[]) result.get("K"), "UTF-8")); + Assert.assertEquals("partition", new String((byte[]) result.get("Q"), "UTF-8")); + Assert.assertEquals(timeStamp, result.get("T")); + Assert.assertEquals("value", new String((byte[]) result.get("V"), "UTF-8")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete(TEST_TABLE, new Object[] { "partitionKey", "partition".getBytes(), + timeStamp }); + } } @Test public void testUpdate() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testPartition", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value".getBytes() }); - long affectedRow = obTableClient.update("testPartition", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - Assert.assertEquals(1, affectedRow); - Map result = obTableClient.get("testPartition", new Object[] { - "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { "K", - "Q", "T", "V", "K_PREFIX" }); - Assert.assertEquals(timeStamp, result.get("T")); - Assert.assertEquals("value1", new String((byte[]) result.get("V"), "UTF-8")); + try { + obTableClient.insert(TEST_TABLE, + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value".getBytes() }); + long affectedRow = obTableClient.update(TEST_TABLE, + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + Assert.assertEquals(1, affectedRow); + Map result = obTableClient.get(TEST_TABLE, new Object[] { + "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { + "K", "Q", "T", "V" }); + Assert.assertEquals(timeStamp, result.get("T")); + Assert.assertEquals("value1", new String((byte[]) result.get("V"), "UTF-8")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete(TEST_TABLE, new Object[] { "partitionKey", "partition".getBytes(), + timeStamp }); + } } @Test public void testReplace() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testPartition", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value".getBytes() }); - long affectedRow = obTableClient.replace("testPartition", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - Assert.assertEquals(2, affectedRow); - Map result = obTableClient.get("testPartition", new Object[] { - "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { "K", - "Q", "T", "V", "K_PREFIX" }); - Assert.assertEquals(timeStamp, result.get("T")); - Assert.assertEquals("value1", new String((byte[]) result.get("V"), "UTF-8")); + try { + obTableClient.insert(TEST_TABLE, + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value".getBytes() }); + long affectedRow = obTableClient.replace(TEST_TABLE, + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + Assert.assertEquals(2, affectedRow); + Map result = obTableClient.get(TEST_TABLE, new Object[] { + "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { + "K", "Q", "T", "V" }); + Assert.assertEquals(timeStamp, result.get("T")); + Assert.assertEquals("value1", new String((byte[]) result.get("V"), "UTF-8")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete(TEST_TABLE, new Object[] { "partitionKey", "partition".getBytes(), + timeStamp }); + } } @Test public void testDelete() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testPartition", + obTableClient.insert(TEST_TABLE, new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { "V" }, new Object[] { "value".getBytes() }); - long affectedRow = obTableClient.delete("testPartition", + long affectedRow = obTableClient.delete(TEST_TABLE, new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }); Assert.assertEquals(1, affectedRow); - Map result = obTableClient.get("testPartition", new Object[] { - "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { "K", - "Q", "T", "V", "K_PREFIX" }); + Map result = obTableClient.get(TEST_TABLE, + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "K", "Q", "T", "V" }); Assert.assertEquals(0, result.size()); } @Test public void testQuery() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testPartition", - new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - obTableClient.insert("testPartition", - new Object[] { "key1_5".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - - TableQuery tableQuery = obTableClient.query("testPartition"); - tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), - timeStamp }, - new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp }); - tableQuery.select("K", "Q", "T", "V", "K_PREFIX"); + cleanTable(TEST_TABLE); + if (!obTableClient.isOdpMode()) { + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_5".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + + TableQuery tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }, new Object[] { "key2_1".getBytes(), "partition".getBytes(), + timeStamp }); + try { + QueryResultSet resultSet = tableQuery.execute(); + int resultCount = 0; + while (resultSet.next()) { + Map value = resultSet.getRow(); + resultCount += 1; + } + Assert.assertFalse(resultSet.next()); + Assert.assertEquals(2, resultCount); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } + tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }, new Object[] { "key1_3".getBytes(), "partition".getBytes(), + timeStamp }); + tableQuery.select("T", "V", "Q", "K"); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + result.next(); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key1_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value1", new String((byte[]) row.get("V"))); + + tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] {"key1_1".getBytes()}, new Object[] {"key1_8".getBytes()}); + tableQuery.setScanRangeColumns("K"); + tableQuery.select("Q", "T", "K", "V"); + result = tableQuery.execute(); + Assert.assertTrue(result.cacheSize() >= 2); + } else { + // TODO: generated column is not supported in ODP mode + timeStamp = System.currentTimeMillis(); + try { + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_2".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value3".getBytes() }); + + TableQuery tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }, new Object[] { "key2_1".getBytes(), "partition".getBytes(), + timeStamp }); + try { + tableQuery.execute(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableUnexpectedException); + + if (obTableClient.isOdpMode()) { + Assert.assertEquals(ResultCodes.OB_ERR_UNEXPECTED.errorCode, + ((ObTableUnexpectedException) e).getErrorCode()); + } else { + Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, + ((ObTableUnexpectedException) e).getErrorCode()); + } + } + + tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }, new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp + 1 }); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + long expTimeStamp[] = { timeStamp, timeStamp + 1 }; + String expValues[] = { "value1", "value2" }; + + for (int i = 0; i < 2; i++) { + Assert.assertEquals(true, result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key1_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(expTimeStamp[i], row.get("T")); + Assert.assertEquals(expValues[i], new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // prefix range scan with scan range columns K + tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, + new Object[] { "key1_1".getBytes() }); + tableQuery.setScanRangeColumns("K"); + tableQuery.select("T", "K", "V"); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + for (int i = 0; i < 2; i++) { + Assert.assertEquals(true, result.next()); + Map row = result.getRow(); + Assert.assertEquals(3, row.size()); + Assert.assertEquals("key1_1", new String((byte[]) row.get("K"))); + Assert.assertEquals(expTimeStamp[i], row.get("T")); + Assert.assertEquals(expValues[i], new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // scan using empty range on key partitioned table + tableQuery = obTableClient.query(TEST_TABLE); + result = tableQuery.execute(); + Assert.assertEquals(3, result.cacheSize()); + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanTable(TEST_TABLE); + } + } + } + + @Test + public void testAsyncQuery() throws Exception { + long timeStamp = System.currentTimeMillis(); + cleanTable(TEST_TABLE); + if (!obTableClient.isOdpMode()) { + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_5".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + + TableQuery tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }, new Object[] { "key2_1".getBytes(), "partition".getBytes(), + timeStamp }); + tableQuery.setBatchSize(2); + try { + QueryResultSet resultSet = tableQuery.asyncExecute(); + int resultCount = 0; + while (resultSet.next()) { + Map value = resultSet.getRow(); + resultCount += 1; + } + Assert.assertFalse(resultSet.next()); + Assert.assertEquals(2, resultCount); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } + + // Test Min - Max Release after addRowkeyElement is completed + tableQuery = obTableClient.query(TEST_TABLE); + tableQuery + .addScanRange(new Object[] { ObObj.getMin(), "partition".getBytes(), timeStamp }, + new Object[] { ObObj.getMax(), "partition".getBytes(), timeStamp }); + tableQuery.setBatchSize(1); + QueryResultSet resultSet = tableQuery.asyncExecute(); + int resultCount = 0; + while (resultSet.next()) { + Map value = resultSet.getRow(); + resultCount += 1; + } + Assert.assertEquals(2, resultCount); + + tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }, new Object[] { "key1_3".getBytes(), "partition".getBytes(), + timeStamp }); + tableQuery.select("T", "V", "Q", "K"); + QueryResultSet result = tableQuery.asyncExecute(); + Assert.assertEquals(1, result.cacheSize()); + result.next(); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key1_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value1", new String((byte[]) row.get("V"))); + + tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange( + new Object[] {"key1_1".getBytes()}, new Object[] {"key1_8".getBytes()}); + tableQuery.select("Q", "T", "K", "V"); + result = tableQuery.asyncExecute(); + Assert.assertTrue(result.cacheSize() >= 2); + } else { + // TODO: generated column is not supported in ODP mode + timeStamp = System.currentTimeMillis(); + try { + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_2".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value3".getBytes() }); + + TableQuery tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }, new Object[] { "key2_1".getBytes(), "partition".getBytes(), + timeStamp }); + tableQuery.setBatchSize(2); + try { + tableQuery.asyncExecute(); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableUnexpectedException); + + if (obTableClient.isOdpMode()) { + Assert.assertEquals(ResultCodes.OB_ERR_UNEXPECTED.errorCode, + ((ObTableUnexpectedException) e).getErrorCode()); + } else { + Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, + ((ObTableUnexpectedException) e).getErrorCode()); + } + } + + tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }, new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp + 1 }); + tableQuery.setBatchSize(5); + QueryResultSet result = tableQuery.asyncExecute(); + Assert.assertEquals(2, result.cacheSize()); + + long expTimeStamp[] = { timeStamp, timeStamp + 1 }; + String expValues[] = { "value1", "value2" }; + + for (int i = 0; i < 2; i++) { + Assert.assertEquals(true, result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key1_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(expTimeStamp[i], row.get("T")); + Assert.assertEquals(expValues[i], new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // prefix range scan with scan range columns K + tableQuery = obTableClient.query(TEST_TABLE); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, + new Object[] { "key1_1".getBytes() }); + tableQuery.setScanRangeColumns("K"); + tableQuery.select("T", "K", "V"); + tableQuery.setBatchSize(1); + result = tableQuery.asyncExecute(); + + for (int i = 0; i < 2; i++) { + Assert.assertEquals(true, result.next()); + Map row = result.getRow(); + Assert.assertEquals(3, row.size()); + Assert.assertEquals("key1_1", new String((byte[]) row.get("K"))); + Assert.assertEquals(expTimeStamp[i], row.get("T")); + Assert.assertEquals(expValues[i], new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // scan using empty range on key partitioned table + tableQuery = obTableClient.query(TEST_TABLE); + result = tableQuery.asyncExecute(); + Assert.assertEquals(3, result.cacheSize()); + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanTable(TEST_TABLE); + } + } + } + + @Test + public void testQueryLocalIndex() throws Exception { + String tableName = "testKey"; + long timeStamp = System.currentTimeMillis(); + try { + obTableClient.insert(tableName, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert(tableName, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp + 2 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert(tableName, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert(tableName, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp + 2 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + + // key partitioned table do not support range query + + // query key2_1 + TableQuery tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { "key2_1".getBytes(), "value0".getBytes() }, + new Object[] { "key2_1".getBytes(), "value9".getBytes() }); + // TODO: do param check, must specify select columns + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.select("K", "V", "T"); + tableQuery.indexName("i1"); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(3, row.size()); + Assert.assertEquals("key2_1", new String((byte[]) row.get("K"))); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + tableQuery.scanOrder(false); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + for (int i = 2; i >= 1; i--) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals("key2_1", new String((byte[]) row.get("K"))); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // query key3_1 + tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { "key3_1".getBytes(), "value0".getBytes() }, + new Object[] { "key3_1".getBytes(), "value9".getBytes() }); + tableQuery.select("K", "Q", "T", "V"); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key3_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + 3 - i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + tableQuery.scanOrder(false); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + for (int i = 2; i >= 1; i--) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key3_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + 3 - i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + + // query key2_1 using K prefix + tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { "key2_1".getBytes() }, + new Object[] { "key2_1".getBytes() }); + tableQuery.setScanRangeColumns("K"); + tableQuery.indexName("i1"); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key2_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete(tableName, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete(tableName, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp + 2 }); + obTableClient.delete(tableName, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete(tableName, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp + 2 }); + } + } + + @Test + public void testAsyncQueryLocalIndex() throws Exception { + long timeStamp = System.currentTimeMillis(); + String tableName = "testKey"; + try { + obTableClient.insert(tableName, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert(tableName, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp + 2 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert(tableName, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + obTableClient.insert(tableName, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp + 2 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + + // key partitioned table do not support range query + + // query key2_1 + TableQuery tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { "key2_1".getBytes(), "value0".getBytes() }, + new Object[] { "key2_1".getBytes(), "value9".getBytes() }); + // TODO: do param check, must specify select columns + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.select("K", "V", "T"); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(1); + QueryResultSet result = tableQuery.asyncExecute(); + Assert.assertEquals(1, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(3, row.size()); + Assert.assertEquals("key2_1", new String((byte[]) row.get("K"))); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + tableQuery.scanOrder(false); + result = tableQuery.asyncExecute(); + Assert.assertEquals(1, result.cacheSize()); + for (int i = 2; i >= 1; i--) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals("key2_1", new String((byte[]) row.get("K"))); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // query key3_1 + tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { "key3_1".getBytes(), "value0".getBytes() }, + new Object[] { "key3_1".getBytes(), "value9".getBytes() }); + tableQuery.select("K", "Q", "T", "V"); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(1); + result = tableQuery.asyncExecute(); + Assert.assertEquals(1, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key3_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + 3 - i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + tableQuery.scanOrder(false); + result = tableQuery.asyncExecute(); + Assert.assertEquals(1, result.cacheSize()); + for (int i = 2; i >= 1; i--) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key3_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + 3 - i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + + // query key2_1 using K prefix + tableQuery = obTableClient.query(tableName); + tableQuery.addScanRange(new Object[] { "key2_1".getBytes() }, + new Object[] { "key2_1".getBytes() }); + tableQuery.setScanRangeColumns("K"); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(1); + result = tableQuery.asyncExecute(); + Assert.assertEquals(1, result.cacheSize()); + for (int i = 1; i <= 2; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("key2_1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + i, row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete(tableName, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete(tableName, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp + 2 }); + obTableClient.delete(tableName, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete(tableName, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp + 2 }); + } + } + + @Test + public void testLargeQuery() throws Exception { + int batchSize = 120; + String key = "key"; + String qualifier = "partition"; + String value = "V"; + int columnSize = 0; try { - tableQuery.execute(); - Assert.fail(); + for (long i = 0; i < batchSize; i++) { + obTableClient.insert(TEST_TABLE, + new Object[] { key.getBytes(), qualifier.getBytes(), i }, new String[] { "V" }, + new Object[] { value.getBytes() }); + } + TableQuery tableQuery = obTableClient.query(TEST_TABLE); + // fixme: generated column is not supported by odp mode + if (obTableClient.isOdpMode()) { + columnSize = 4; + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[] { "key".getBytes() }, + new Object[] { "key".getBytes() }); + if (ObGlobal.obVsnMajor() < 4) { + tableQuery.select("K", "Q", "T", "V"); + } + } else { + // todo: scan_range_columns cannot be used for routing + columnSize = 5; + tableQuery.addScanRange(new Object[] { "key".getBytes(), "a".getBytes(), + Long.MIN_VALUE }, new Object[] { "key".getBytes(), "z".getBytes(), + Long.MAX_VALUE }); + if (ObGlobal.obVsnMajor() < 4) { + tableQuery.select("K", "Q", "T", "V", "K_PREFIX"); + } + } + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(batchSize, result.cacheSize()); + + for (long i = 0; i < batchSize; i++) { + Assert.assertEquals(true, result.next()); + Map row = result.getRow(); + Assert.assertEquals(columnSize, row.size()); + Assert.assertEquals(key, new String((byte[]) row.get("K"))); + Assert.assertEquals(qualifier, new String((byte[]) row.get("Q"))); + Assert.assertEquals(i, row.get("T")); + Assert.assertEquals(value, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); } catch (Exception e) { - Assert.assertTrue(e instanceof ObTablePartitionConsistentException); + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (long i = 0; i < batchSize; i++) { + obTableClient.delete(TEST_TABLE, + new Object[] { key.getBytes(), qualifier.getBytes(), i }); + } + } + } + + @Test + public void testLargeAsyncQuery() throws Exception { + int batchSize = 120; + String key = "key"; + String qualifier = "partition"; + String value = "V"; + int columnSize = 0; + try { + for (long i = 0; i < batchSize; i++) { + obTableClient.insert(TEST_TABLE, + new Object[] { key.getBytes(), qualifier.getBytes(), i }, new String[] { "V" }, + new Object[] { value.getBytes() }); + } + TableQuery tableQuery = obTableClient.query(TEST_TABLE); + // fixme: generated column is not supported by odp mode + if (obTableClient.isOdpMode()) { + columnSize = 4; + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[] { "key".getBytes() }, + new Object[] { "key".getBytes() }); + tableQuery.setBatchSize(3); + if (ObGlobal.obVsnMajor() < 4) { + tableQuery.select("K", "Q", "T", "V"); + } + } else { + // todo: scan_range_columns cannot be used for routing + columnSize = 5; + tableQuery.addScanRange(new Object[] { "key".getBytes(), "a".getBytes(), + Long.MIN_VALUE }, new Object[] { "key".getBytes(), "z".getBytes(), + Long.MAX_VALUE }); + tableQuery.setBatchSize(3); + tableQuery.setOperationTimeout(100); + if (ObGlobal.obVsnMajor() < 4) { + tableQuery.select("K", "Q", "T", "V", "K_PREFIX"); + } + } + QueryResultSet result = tableQuery.asyncExecute(); + + for (long i = 0; i < batchSize; i++) { + Assert.assertEquals(true, result.next()); + Map row = result.getRow(); + Assert.assertEquals(columnSize, row.size()); + Assert.assertEquals(key, new String((byte[]) row.get("K"))); + Assert.assertEquals(qualifier, new String((byte[]) row.get("Q"))); + Assert.assertEquals(i, row.get("T")); + Assert.assertEquals(value, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (long i = 0; i < batchSize; i++) { + obTableClient.delete(TEST_TABLE, + new Object[] { key.getBytes(), qualifier.getBytes(), i }); + } } - tableQuery = obTableClient.query("testPartition"); - tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes(), - timeStamp }, - new Object[] { "key1_3".getBytes(), "partition".getBytes(), timeStamp }); - tableQuery.select("K", "Q", "T", "V"); - QueryResultSet result = tableQuery.execute(); - Assert.assertEquals(1, result.cacheSize()); - result.next(); - Map row = result.getRow(); - Assert.assertEquals("key1_1", new String((byte[]) row.get("K"))); - Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); - Assert.assertEquals(timeStamp, row.get("T")); - Assert.assertEquals("value1", new String((byte[]) row.get("V"))); - - tableQuery = obTableClient.query("testPartition"); - tableQuery.addScanRange( - new Object[] { "key1_1".getBytes(), ObObj.getMin(), ObObj.getMin() }, new Object[] { - "key1_8".getBytes(), ObObj.getMax(), ObObj.getMax() }); - tableQuery.select("K", "Q", "T", "V"); - result = tableQuery.execute(); - - Assert.assertTrue(result.cacheSize() >= 2); } @Test public void testBatch() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testPartition", - new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - obTableClient.insert("testPartition", - new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - TableBatchOps tableBatchOps = obTableClient.batch("testPartition"); - tableBatchOps - .delete(new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }); - tableBatchOps.insert( - new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - tableBatchOps.replace( - new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - List batchResult = tableBatchOps.execute(); - Assert.assertEquals(3, batchResult.size()); - Assert.assertEquals(1L, batchResult.get(0)); - Assert.assertEquals(1L, batchResult.get(1)); - Assert.assertEquals(2L, batchResult.get(2)); - - Map getResult = obTableClient.get("testPartition", - new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, new String[] { - "K", "Q", "T", "V" }); + try { + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert(TEST_TABLE, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + TableBatchOps tableBatchOps = obTableClient.batch(TEST_TABLE); + tableBatchOps.delete(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }); + tableBatchOps.insert(new Object[] { "key2_1".getBytes(), "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value2".getBytes() }); + tableBatchOps.replace(new Object[] { "key3_1".getBytes(), "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value2".getBytes() }); + List batchResult = tableBatchOps.execute(); + Assert.assertEquals(3, batchResult.size()); + Assert.assertEquals(1L, batchResult.get(0)); + Assert.assertEquals(1L, batchResult.get(1)); + Assert.assertEquals(2L, batchResult.get(2)); - Assert.assertEquals(0, getResult.size()); + Map getResult = obTableClient.get(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "K", "Q", "T", "V" }); - getResult = obTableClient.get("testPartition", new Object[] { "key2_1".getBytes(), - "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); + Assert.assertEquals(0, getResult.size()); - Assert.assertEquals(4, getResult.size()); + getResult = obTableClient.get(TEST_TABLE, new Object[] { "key2_1".getBytes(), + "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); - Assert.assertEquals("key2_1", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + Assert.assertEquals(4, getResult.size()); - getResult = obTableClient.get("testPartition", new Object[] { "key3_1".getBytes(), - "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); + Assert.assertEquals("key2_1", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); - Assert.assertEquals(4, getResult.size()); + getResult = obTableClient.get(TEST_TABLE, new Object[] { "key3_1".getBytes(), + "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); - Assert.assertEquals("key3_1", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + Assert.assertEquals(4, getResult.size()); + + Assert.assertEquals("key3_1", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }); + obTableClient.delete(TEST_TABLE, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp }); + obTableClient.delete(TEST_TABLE, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }); + } } @Test public void testBatchConcurrent() throws Exception { long timeStamp = System.currentTimeMillis(); + // This opetion is useless in ODP mode obTableClient.setRuntimeBatchExecutor(Executors.newFixedThreadPool(3)); - obTableClient.insert("testPartition", - new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - obTableClient.insert("testPartition", - new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - TableBatchOps tableBatchOps = obTableClient.batch("testPartition"); - tableBatchOps - .delete(new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }); - tableBatchOps.insert( - new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - tableBatchOps.replace( - new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - List batchResult = tableBatchOps.execute(); - Assert.assertEquals(3, batchResult.size()); - Assert.assertEquals(1L, batchResult.get(0)); - Assert.assertEquals(1L, batchResult.get(1)); - Assert.assertEquals(2L, batchResult.get(2)); - - Map getResult = obTableClient.get("testPartition", - new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, new String[] { - "K", "Q", "T", "V" }); + try { + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert(TEST_TABLE, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + TableBatchOps tableBatchOps = obTableClient.batch(TEST_TABLE); + tableBatchOps.delete(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }); + tableBatchOps.insert(new Object[] { "key2_1".getBytes(), "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value2".getBytes() }); + tableBatchOps.replace(new Object[] { "key3_1".getBytes(), "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value2".getBytes() }); + List batchResult = tableBatchOps.execute(); + Assert.assertEquals(3, batchResult.size()); + Assert.assertEquals(1L, batchResult.get(0)); + Assert.assertEquals(1L, batchResult.get(1)); + Assert.assertEquals(2L, batchResult.get(2)); + + Map getResult = obTableClient.get(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(0, getResult.size()); - Assert.assertEquals(0, getResult.size()); + getResult = obTableClient.get(TEST_TABLE, new Object[] { "key2_1".getBytes(), + "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); - getResult = obTableClient.get("testPartition", new Object[] { "key2_1".getBytes(), - "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); + Assert.assertEquals(4, getResult.size()); - Assert.assertEquals(4, getResult.size()); + Assert.assertEquals("key2_1", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); - Assert.assertEquals("key2_1", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + getResult = obTableClient.get(TEST_TABLE, new Object[] { "key3_1".getBytes(), + "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); - getResult = obTableClient.get("testPartition", new Object[] { "key3_1".getBytes(), - "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); + Assert.assertEquals(4, getResult.size()); - Assert.assertEquals(4, getResult.size()); + Assert.assertEquals("key3_1", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); - Assert.assertEquals("key3_1", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }); + obTableClient.delete(TEST_TABLE, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp }); + obTableClient.delete(TEST_TABLE, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }); + } } @Test public void testBatchConcurrentWithPriority() throws Exception { long timeStamp = System.currentTimeMillis(); + // This following two option in ODP mode is useless. ThreadLocalMap.setProcessHighPriority(); obTableClient.setRuntimeBatchExecutor(Executors.newFixedThreadPool(3)); - obTableClient.insert("testPartition", - new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - obTableClient.insert("testPartition", - new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - TableBatchOps tableBatchOps = obTableClient.batch("testPartition"); - tableBatchOps - .delete(new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }); - tableBatchOps.insert( - new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - tableBatchOps.replace( - new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - List batchResult = tableBatchOps.execute(); - Assert.assertEquals(3, batchResult.size()); - Assert.assertEquals(1L, batchResult.get(0)); - Assert.assertEquals(1L, batchResult.get(1)); - Assert.assertEquals(2L, batchResult.get(2)); - - Map getResult = obTableClient.get("testPartition", - new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, new String[] { - "K", "Q", "T", "V" }); + try { + obTableClient.insert(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert(TEST_TABLE, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + TableBatchOps tableBatchOps = obTableClient.batch(TEST_TABLE); + tableBatchOps.delete(new Object[] { "key1_1".getBytes(), "partition".getBytes(), + timeStamp }); + tableBatchOps.insert(new Object[] { "key2_1".getBytes(), "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value2".getBytes() }); + tableBatchOps.replace(new Object[] { "key3_1".getBytes(), "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value2".getBytes() }); + List batchResult = tableBatchOps.execute(); + Assert.assertEquals(3, batchResult.size()); + Assert.assertEquals(1L, batchResult.get(0)); + Assert.assertEquals(1L, batchResult.get(1)); + Assert.assertEquals(2L, batchResult.get(2)); + + Map getResult = obTableClient.get(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "K", "Q", "T", "V" }); - Assert.assertEquals(0, getResult.size()); + Assert.assertEquals(0, getResult.size()); - getResult = obTableClient.get("testPartition", new Object[] { "key2_1".getBytes(), - "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); + getResult = obTableClient.get(TEST_TABLE, new Object[] { "key2_1".getBytes(), + "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); - Assert.assertEquals(4, getResult.size()); + Assert.assertEquals(4, getResult.size()); - Assert.assertEquals("key2_1", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + Assert.assertEquals("key2_1", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); - getResult = obTableClient.get("testPartition", new Object[] { "key3_1".getBytes(), - "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); + getResult = obTableClient.get(TEST_TABLE, new Object[] { "key3_1".getBytes(), + "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); - Assert.assertEquals(4, getResult.size()); + Assert.assertEquals(4, getResult.size()); - Assert.assertEquals("key3_1", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + Assert.assertEquals("key3_1", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete(TEST_TABLE, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }); + obTableClient.delete(TEST_TABLE, + new Object[] { "key2_1".getBytes(), "partition".getBytes(), timeStamp }); + obTableClient.delete(TEST_TABLE, + new Object[] { "key3_1".getBytes(), "partition".getBytes(), timeStamp }); + } } + @Test + public void testPartitionLocation() throws Exception { + obTableClient.setRunningMode(ObTableClient.RunningMode.NORMAL); + String testTable = "testPartitionKeyComplex"; + obTableClient.addRowKeyElement(testTable, + new String[] { "c0", "c1", "c2", "c3", "c4", "c5" }); + try { + cleanTable(testTable); + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + for (int i = 0; i < 64; i++) { + byte c0 = (byte) i; + int c1 = i * (i + 1) * (i + 2); + long c2 = i * (i + 1) * (i + 2); + String c3 = generateRandomStringByUUID(10); + String c4 = generateRandomStringByUUID(5) + c2 + generateRandomStringByUUID(5); + String c5 = generateRandomStringByUUID(5) + c3 + generateRandomStringByUUID(5); + + // use sql to insert data + statement.execute("insert into " + testTable + + "(c0, c1, c2, c3, c4, c5, c6) values (" + c0 + "," + c1 + "," + + c2 + ",'" + c3 + "','" + c4 + "','" + c5 + "'," + "'value')"); + + // get data by obkv interface + Map result = obTableClient.get(testTable, new Object[] { c0, c1, + c2, c3.getBytes(), c4, c5 }, new String[] { "c0", "c1", "c2", "c3", "c4", + "c5", "c6" }); + Assert.assertEquals(7, result.size()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanTable(testTable); + } + } + + @Test + public void testTwoPartitionQuery() throws Exception { + obTableClient.setRunningMode(ObTableClient.RunningMode.NORMAL); + String testTable = "testPartitionKeyComplex"; + obTableClient.addRowKeyElement(testTable, + new String[] { "c0", "c1", "c2", "c3", "c4", "c5" }); + try { + cleanTable(testTable); + byte c0 = (byte) 0; + int c1 = 10001; + long c2 = 100001; + String c3 = generateRandomStringByUUID(10); + String c4 = generateRandomStringByUUID(5) + c2 + generateRandomStringByUUID(5); + String c5 = generateRandomStringByUUID(5) + c3 + generateRandomStringByUUID(5); + TableQuery query = obTableClient + .query(testTable) + .addScanRange(new Object[] { c0, c1, c2, c3, c4, c5 }, + new Object[] { c0, c1, c2, c3, c4, c5 }) + .select("c1", "c2", "c3", "c4", "c5", "c6") + .setFilter( + andList(compareVal(ObCompareOp.GE, "c2", 0), + compareVal(ObCompareOp.LE, "c3", 0))); + QueryResultSet resultSet = query.execute(); + Assert.assertEquals(0, resultSet.cacheSize()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanTable(testTable); + } + } + + @Test + public void testTwoPartitionAsyncQuery() throws Exception { + obTableClient.setRunningMode(ObTableClient.RunningMode.NORMAL); + String testTable = "testPartitionKeyComplex"; + obTableClient.addRowKeyElement(testTable, + new String[] { "c0", "c1", "c2", "c3", "c4", "c5" }); + try { + cleanTable(testTable); + byte c0 = (byte) 0; + int c1 = 10001; + long c2 = 100001; + String c3 = generateRandomStringByUUID(10); + String c4 = generateRandomStringByUUID(5) + c2 + generateRandomStringByUUID(5); + String c5 = generateRandomStringByUUID(5) + c3 + generateRandomStringByUUID(5); + TableQuery query = obTableClient + .query(testTable) + .addScanRange(new Object[] { c0, c1, c2, c3, c4, c5 }, + new Object[] { c0, c1, c2, c3, c4, c5 }) + .select("c1", "c2", "c3", "c4", "c5", "c6") + .setBatchSize(1) + .setFilter( + andList(compareVal(ObCompareOp.GE, "c2", 0), + compareVal(ObCompareOp.LE, "c3", 0))); + QueryResultSet resultSet = query.asyncExecute(); + Assert.assertEquals(0, resultSet.cacheSize()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanTable(testTable); + } + } + + private void insertMultiPartition(ObTableClient obTableClient, int count, String tableName) + throws Exception { + int num = 1; + if (count <= 5) { + while (num <= count) { + String uid = "GU1" + String.format("%04d", num); + String object_id = "GO1" + String.format("%04d", num); + String ver_oid = "GV1" + String.format("%04d", num); + String data_id = "GD1" + String.format("%04d", num); + Row rowKey = row(colVal("id", Integer.toUnsignedLong(num)), colVal("uid", uid), + colVal("object_id", object_id)); + Row rows = row(colVal("type", num), colVal("ver_oid", ver_oid), + colVal("ver_ts", System.currentTimeMillis()), colVal("data_id", data_id)); + MutationResult result = obTableClient.insert(tableName).setRowKey(rowKey) + .addMutateRow(rows).execute(); + Assert.assertEquals(1, result.getAffectedRows()); + num++; + } + } else { + BatchOperation batchOperation = obTableClient.batchOperation(tableName); + while (num <= count) { + String uid = "GU1" + String.format("%04d", num); + String object_id = "GO1" + String.format("%04d", num); + String ver_oid = "GV1" + String.format("%04d", num); + String data_id = "GD1" + String.format("%04d", num); + Row rowKey = row(colVal("id", Integer.toUnsignedLong(num)), colVal("uid", uid), + colVal("object_id", object_id)); + Row rows = row(colVal("type", num), colVal("ver_oid", ver_oid), + colVal("ver_ts", System.currentTimeMillis()), colVal("data_id", data_id)); + batchOperation.addOperation(insert().setRowKey(rowKey).addMutateRow(rows)); + num++; + if (num % 100 == 0 || num == count + 1) { + BatchOperationResult result = batchOperation.execute(); + batchOperation = obTableClient.batchOperation(tableName); + } + } + } + } + + @Test + public void testHashKeyQuery() throws Exception { + String tableName = "hash_key_sub_part"; + try { + normalClient.addRowKeyElement(tableName, new String[] { "id", "uid" }); + insertMultiPartition(normalClient, 20, tableName); + TableQuery query = normalClient.query(tableName); + QueryResultSet resultSet; + query.clear(); + query.addScanRange(new Object[] { 1L, "GU20004", "GO10001" }, new Object[] { 30L, + "GU20004", "GO10030" }); + resultSet = query.execute(); + + int resultCount = 0; + while (resultSet.next()) { + Map value = resultSet.getRow(); + resultCount += 1; + } + Assert.assertFalse(resultSet.next()); + Assert.assertEquals(19, resultCount); + } finally { + cleanTable(tableName); + } + } + + @Test + public void testHashKeyAsyncQuery() throws Exception { + String tableName = "hash_key_sub_part"; + try { + normalClient.addRowKeyElement(tableName, new String[] { "id", "uid" }); + insertMultiPartition(normalClient, 20, tableName); + TableQuery query = normalClient.query(tableName); + QueryResultSet resultSet; + query.clear(); + query.addScanRange(new Object[] { 1L, "GU20004", "GO10001" }, new Object[] { 30L, + "GU20004", "GO10030" }); + query.setBatchSize(1); + resultSet = query.asyncExecute(); + + int resultCount = 0; + while (resultSet.next()) { + Map value = resultSet.getRow(); + resultCount += 1; + } + Assert.assertFalse(resultSet.next()); + Assert.assertEquals(19, resultCount); + } finally { + cleanTable(tableName); + } + } + + @Test + public void testFirstPartStartEndKeys() throws Exception { + // CREATE TABLEGROUP test_start_end_keys_key SHARDING = 'ADAPTIVE'; + // CREATE TABLE `test_start_end_keys$family_key` ( + // `K` varbinary(1024) NOT NULL, + // `Q` varbinary(256) NOT NULL, + // `T` bigint(20) NOT NULL, + // `V` varbinary(1024) DEFAULT NULL, + // PRIMARY KEY (`K`, `Q`, `T`) + // ) partition by key(K) partitions 17 + // TABLEGROUP = test_start_end_keys_key; + try { + byte[][][] keyFirstPartStartKeys = obTableClient + .getFirstPartStartKeys("test_start_end_keys$family_key"); + byte[][][] keyFirstPartEndKeys = obTableClient + .getFirstPartEndKeys("test_start_end_keys$family_key"); + Assert.assertArrayEquals(keyFirstPartStartKeys, keyFirstPartEndKeys); + Assert.assertEquals(1, keyFirstPartStartKeys.length); + Assert.assertEquals(1, keyFirstPartStartKeys[0].length); + Assert.assertEquals(0, keyFirstPartStartKeys[0][0].length); + } catch (Exception e) { + fail(); + } + } + + @Test + public void testHBaseStartEndKeys() throws Exception { + // CREATE TABLEGROUP test_start_end_keys_key SHARDING = 'ADAPTIVE'; + // CREATE TABLE `test_start_end_keys$family_key` ( + // `K` varbinary(1024) NOT NULL, + // `Q` varbinary(256) NOT NULL, + // `T` bigint(20) NOT NULL, + // `V` varbinary(1024) DEFAULT NULL, + // PRIMARY KEY (`K`, `Q`, `T`) + // ) partition by key(K) partitions 17 + // TABLEGROUP = test_start_end_keys_key; + + try { + byte[][] keyHBaseStartKeys = obTableClient + .getHBaseTableStartKeys("test_start_end_keys_key"); + byte[][] keyHBaseEndKeys = obTableClient + .getHBaseTableEndKeys("test_start_end_keys_key"); + Assert.assertArrayEquals(keyHBaseStartKeys, keyHBaseEndKeys); + Assert.assertEquals(1, keyHBaseStartKeys.length); + Assert.assertEquals(0, keyHBaseStartKeys[0].length); + } catch (Exception e) { + fail(); + } + } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionRangeTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionRangeTest.java index 51fdd8f9..df3f8d4e 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionRangeTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientPartitionRangeTest.java @@ -22,14 +22,22 @@ import com.alipay.oceanbase.rpc.table.api.TableQuery; import com.alipay.oceanbase.rpc.threadlocal.ThreadLocalMap; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.util.List; -import java.util.Map; +import java.sql.Connection; +import java.sql.Statement; +import java.text.SimpleDateFormat; +import java.util.*; import java.util.concurrent.Executors; +import static com.alipay.oceanbase.rpc.util.ObTableClientTestUtil.cleanTable; +import static com.alipay.oceanbase.rpc.util.ObTableClientTestUtil.generateRandomStringByUUID; +import static java.lang.StrictMath.abs; +import static org.junit.Assert.fail; + public class ObTableClientPartitionRangeTest { private ObTableClient obTableClient; @@ -53,77 +61,118 @@ public void setUp() throws Exception { this.obTableClient = obTableClient; } + @After + public void close() throws Exception { + if (null != this.obTableClient) { + ((ObTableClient) this.obTableClient).close(); + } + } + @Test public void testInsert() throws Exception { long timestamp = System.currentTimeMillis(); - long affectRow = obTableClient.insert("testRange", new Object[] { - "partitionKey".getBytes(), "partition".getBytes(), timestamp }, - new String[] { "V" }, new Object[] { "aa".getBytes() }); - Assert.assertEquals(1, affectRow); - - affectRow = obTableClient.insertOrUpdate("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timestamp }, - new String[] { "V" }, new Object[] { "bb".getBytes() }); - Assert.assertEquals(1, affectRow); - - Map result = obTableClient.get("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timestamp }, - new String[] { "K", "Q", "T", "V" }); - Assert.assertEquals("partitionKey", new String((byte[]) result.get("K"), "UTF-8")); - Assert.assertEquals("partition", new String((byte[]) result.get("Q"), "UTF-8")); - Assert.assertEquals(timestamp, result.get("T")); - Assert.assertEquals("bb", new String((byte[]) result.get("V"), "UTF-8")); + try { + long affectRow = obTableClient.insert("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timestamp }, + new String[] { "V" }, new Object[] { "aa".getBytes() }); + Assert.assertEquals(1, affectRow); + + affectRow = obTableClient.insertOrUpdate("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timestamp }, + new String[] { "V" }, new Object[] { "bb".getBytes() }); + Assert.assertEquals(1, affectRow); + + Map result = obTableClient.get("testRange", new Object[] { + "partitionKey".getBytes(), "partition".getBytes(), timestamp }, new String[] { + "K", "Q", "T", "V" }); + Assert.assertEquals("partitionKey", new String((byte[]) result.get("K"), "UTF-8")); + Assert.assertEquals("partition", new String((byte[]) result.get("Q"), "UTF-8")); + Assert.assertEquals(timestamp, result.get("T")); + Assert.assertEquals("bb", new String((byte[]) result.get("V"), "UTF-8")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timestamp }); + obTableClient.delete("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timestamp }); + } } @Test public void testGet() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value".getBytes() }); - Map result = obTableClient.get("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "K", "Q", "T", "V" }); - Assert.assertEquals("partitionKey", new String((byte[]) result.get("K"), "UTF-8")); - Assert.assertEquals("partition", new String((byte[]) result.get("Q"), "UTF-8")); - Assert.assertEquals(timeStamp, result.get("T")); - Assert.assertEquals("value", new String((byte[]) result.get("V"), "UTF-8")); + try { + obTableClient.insert("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value".getBytes() }); + Map result = obTableClient.get("testRange", new Object[] { + "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { + "K", "Q", "T", "V" }); + Assert.assertEquals("partitionKey", new String((byte[]) result.get("K"), "UTF-8")); + Assert.assertEquals("partition", new String((byte[]) result.get("Q"), "UTF-8")); + Assert.assertEquals(timeStamp, result.get("T")); + Assert.assertEquals("value", new String((byte[]) result.get("V"), "UTF-8")); + } catch (Exception e) { + + } finally { + obTableClient.delete("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }); + } } @Test public void testUpdate() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value".getBytes() }); - long affectedRow = obTableClient.update("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - Assert.assertEquals(1, affectedRow); - Map result = obTableClient.get("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "K", "Q", "T", "V" }); - Assert.assertEquals(timeStamp, result.get("T")); - Assert.assertEquals("value1", new String((byte[]) result.get("V"), "UTF-8")); + try { + obTableClient.insert("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value".getBytes() }); + long affectedRow = obTableClient.update("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + Assert.assertEquals(1, affectedRow); + Map result = obTableClient.get("testRange", new Object[] { + "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { + "K", "Q", "T", "V" }); + Assert.assertEquals(timeStamp, result.get("T")); + Assert.assertEquals("value1", new String((byte[]) result.get("V"), "UTF-8")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }); + } + } @Test public void testReplace() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value".getBytes() }); - long affectedRow = obTableClient.replace("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - Assert.assertEquals(2, affectedRow); - Map result = obTableClient.get("testRange", - new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, - new String[] { "K", "Q", "T", "V" }); - Assert.assertEquals(timeStamp, result.get("T")); - Assert.assertEquals("value1", new String((byte[]) result.get("V"), "UTF-8")); + try { + obTableClient.insert("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value".getBytes() }); + long affectedRow = obTableClient.replace("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + Assert.assertEquals(2, affectedRow); + Map result = obTableClient.get("testRange", new Object[] { + "partitionKey".getBytes(), "partition".getBytes(), timeStamp }, new String[] { + "K", "Q", "T", "V" }); + Assert.assertEquals(timeStamp, result.get("T")); + Assert.assertEquals("value1", new String((byte[]) result.get("V"), "UTF-8")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", + new Object[] { "partitionKey".getBytes(), "partition".getBytes(), timeStamp }); + } } @Test @@ -144,169 +193,789 @@ public void testDelete() throws Exception { @Test public void testQuery() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testRange", new Object[] { "ah", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - obTableClient.insert("testRange", new Object[] { "xh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - - TableQuery tableQuery = obTableClient.query("testRange"); - tableQuery.addScanRange(new Object[] { "ah", "partition".getBytes(), timeStamp }, - new Object[] { "az", "partition".getBytes(), timeStamp }); - tableQuery.select("K", "Q", "T", "V"); - QueryResultSet result = tableQuery.execute(); - Assert.assertEquals(1, result.cacheSize()); - - tableQuery = obTableClient.query("testRange"); - tableQuery.addScanRange(new Object[] { "ah", "partition".getBytes(), timeStamp }, - new Object[] { "xz", "partition".getBytes(), timeStamp }); - tableQuery.select("K", "Q", "T", "V"); - - result = tableQuery.execute(); - Assert.assertTrue(result.cacheSize() >= 2); - // server not supported - // tableQuery = obTableClient.query("testRange"); - // tableQuery.addScanRange(new Object[]{timeStamp + "", "partition".getBytes(), timeStamp}, new Object[]{timeStamp + 10 + "", "partition".getBytes(), timeStamp}); - // tableQuery.select("K", "Q", "T", "V"); - // result = tableQuery.execute(); - // Assert.assertEquals(2, result.size()); + try { + // p0 + obTableClient.insert("testRange", + new Object[] { "0", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + // p1 + obTableClient.insert("testRange", + new Object[] { "ah", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testRange", + new Object[] { "uh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + // p2 + obTableClient.insert("testRange", + new Object[] { "xh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + + // single partition query + TableQuery tableQuery = obTableClient.query("testRange"); + tableQuery.addScanRange(new Object[] { "ah", "partition".getBytes(), timeStamp }, + new Object[] { "az", "partition".getBytes(), timeStamp }); + tableQuery.select("Q", "T", "K", "V"); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("ah", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value1", new String((byte[]) row.get("V"))); + + // multiply partition query + tableQuery = obTableClient.query("testRange"); + tableQuery.addScanRange(new Object[] { "0", "partition".getBytes(), timeStamp }, + new Object[] { "xz", "partition".getBytes(), timeStamp }); + tableQuery.select("Q", "T", "K", "V"); + result = tableQuery.execute(); + Assert.assertEquals(4, result.cacheSize()); + + // sort result by K + List> resultList = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + resultList.add(result.getRow()); + } + Assert.assertFalse(result.next()); + Collections.sort(resultList, new Comparator>() { + @Override + public int compare(Map o1, Map o2) { + return new String((byte[])o1.get("K")).compareTo(new String((byte[])o2.get("K"))); + } + }); + String[] orderedKeys = {"0", "ah", "uh", "xh"}; + for (int i = 0; i < 4; i++) { + Assert.assertEquals(orderedKeys[i], new String((byte[])resultList.get(i).get("K"))); + Assert.assertEquals("partition", new String((byte[]) resultList.get(i).get("Q"))); + Assert.assertEquals(timeStamp, resultList.get(i).get("T")); + Assert.assertEquals("value1", new String((byte[]) resultList.get(i).get("V"))); + } + Assert.assertFalse(result.next()); + + // single partition query using prefix + tableQuery = obTableClient.query("testRange"); + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[] { "ah" }, new Object[] { "az" }); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + Assert.assertTrue(result.next()); + row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("ah", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value1", new String((byte[]) row.get("V"))); + + // query use empty scan range + tableQuery = obTableClient.query("testRange"); + result = tableQuery.execute(); + Assert.assertEquals(4, result.cacheSize()); + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", new Object[] { "0", "partition".getBytes(), timeStamp }); + obTableClient.delete("testRange", new Object[] { "ah", "partition".getBytes(), timeStamp }); + obTableClient.delete("testRange", new Object[] { "uh", "partition".getBytes(), timeStamp }); + obTableClient.delete("testRange", new Object[] { "xh", "partition".getBytes(), timeStamp }); + } + } + + @Test + public void testAsyncQuery() throws Exception { + long timeStamp = System.currentTimeMillis(); + try { + // p0 + obTableClient.insert("testRange", + new Object[] { "0", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + // p1 + obTableClient.insert("testRange", + new Object[] { "ah", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testRange", + new Object[] { "uh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + // p2 + obTableClient.insert("testRange", + new Object[] { "xh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + + // single partition query + TableQuery tableQuery = obTableClient.query("testRange"); + tableQuery.addScanRange(new Object[] { "ah", "partition".getBytes(), timeStamp }, + new Object[] { "az", "partition".getBytes(), timeStamp }); + tableQuery.setBatchSize(2); + tableQuery.select("Q", "T", "K", "V"); + QueryResultSet result = tableQuery.asyncExecute(); + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("ah", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value1", new String((byte[]) row.get("V"))); + + // multiply partition query + tableQuery = obTableClient.query("testRange"); + tableQuery.addScanRange(new Object[] { "0", "partition".getBytes(), timeStamp }, + new Object[] { "xz", "partition".getBytes(), timeStamp }); + tableQuery.select("Q", "T", "K", "V"); + tableQuery.setBatchSize(2); + result = tableQuery.asyncExecute(); + + // sort result by K + List> resultList = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + resultList.add(result.getRow()); + } + Assert.assertFalse(result.next()); + Collections.sort(resultList, new Comparator>() { + @Override + public int compare(Map o1, Map o2) { + return new String((byte[])o1.get("K")).compareTo(new String((byte[])o2.get("K"))); + } + }); + String[] orderedKeys = {"0", "ah", "uh", "xh"}; + for (int i = 0; i < 4; i++) { + Assert.assertEquals(orderedKeys[i], new String((byte[])resultList.get(i).get("K"))); + Assert.assertEquals("partition", new String((byte[]) resultList.get(i).get("Q"))); + Assert.assertEquals(timeStamp, resultList.get(i).get("T")); + Assert.assertEquals("value1", new String((byte[]) resultList.get(i).get("V"))); + } + Assert.assertFalse(result.next()); + + if (obTableClient.isOdpMode()) { + // single partition query using prefix + tableQuery = obTableClient.query("testRange"); + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[]{"ah"}, + new Object[]{"az"}); + tableQuery.setBatchSize(2); + result = tableQuery.asyncExecute(); + Assert.assertEquals(1, result.cacheSize()); + Assert.assertTrue(result.next()); + row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("ah", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp, row.get("T")); + Assert.assertEquals("value1", new String((byte[]) row.get("V"))); + + tableQuery = obTableClient.query("testRange"); + tableQuery.setBatchSize(2); + result = tableQuery.asyncExecute(); + Assert.assertEquals(4, result.cacheSize()); + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", new Object[] { "0", "partition".getBytes(), timeStamp }); + obTableClient.delete("testRange", new Object[] { "ah", "partition".getBytes(), timeStamp }); + obTableClient.delete("testRange", new Object[] { "uh", "partition".getBytes(), timeStamp }); + obTableClient.delete("testRange", new Object[] { "xh", "partition".getBytes(), timeStamp }); + } + } + + @Test + public void testQueryLocalIndex() throws Exception { + long timeStamp = System.currentTimeMillis(); + try { + // p1 + obTableClient.insert("testRange", + new Object[] { "a1", "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testRange", + new Object[] { "a1", "partition".getBytes(), timeStamp + 2}, + new String[] { "V" }, new Object[] { "value3".getBytes() }); + obTableClient.insert("testRange", + new Object[] { "a1", "partition".getBytes(), timeStamp + 3}, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + // p2 + obTableClient.insert("testRange", + new Object[] { "x1", "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + + // single partition query with index + TableQuery tableQuery = obTableClient.query("testRange"); + tableQuery.addScanRange(new Object[] { "a1", "value1".getBytes() }, + new Object[] { "a1", "value9".getBytes() }); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(3, result.cacheSize()); + + int[] tsDelta = {1, 3, 2}; + for (int i = 1; i <= 3; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("a1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + tsDelta[i - 1], row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // query with index backward + tableQuery.scanOrder(false); + result = tableQuery.execute(); + Assert.assertEquals(3, result.cacheSize()); + for (int i = 3; i >= 1; i--) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("a1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + tsDelta[i - 1], row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // single partition query with index prefix + tableQuery = obTableClient.query("testRange"); + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[] { "a1" }, + new Object[] { "a1" }); + tableQuery.select("K", "T", "V"); + tableQuery.indexName("i1"); + result = tableQuery.execute(); + Assert.assertEquals(3, result.cacheSize()); + + for (int i = 1; i <= 3; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(3, row.size()); + Assert.assertEquals("a1", new String((byte[]) row.get("K"))); + Assert.assertEquals(timeStamp + tsDelta[i - 1], row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // multiply partition query with index + tableQuery = obTableClient.query("testRange"); + tableQuery.addScanRange(new Object[] { "a0", "value1" }, new Object[] { "z9", "value9" } ); + tableQuery.select("K", "V", "Q", "T"); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + result = tableQuery.execute(); + Assert.assertEquals(4, result.cacheSize()); + + // sort result by K, T + List> resultList = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + resultList.add(result.getRow()); + } + Assert.assertFalse(result.next()); + Collections.sort(resultList, new Comparator>() { + @Override + public int compare(Map o1, Map o2) { + int firstCmp = new String((byte[])o1.get("K")).compareTo(new String((byte[])o2.get("K"))); + if (firstCmp == 0) { + return (int)(((long)o1.get("T")) - ((long)o2.get("T"))); + } + return firstCmp; + } + }); + String[] orderedKeys = {"a1", "a1", "a1", "x1"}; + int[] orderedDeltaTs = {1, 2, 3, 1}; + String[] orderedValues = {"value1", "value3", "value2", "value1"}; + for (int i = 0; i < 4; i++) { + Assert.assertEquals(orderedKeys[i], new String((byte[])resultList.get(i).get("K"))); + Assert.assertEquals("partition", new String((byte[]) resultList.get(i).get("Q"))); + Assert.assertEquals(timeStamp + orderedDeltaTs[i], resultList.get(i).get("T")); + Assert.assertEquals(orderedValues[i], new String((byte[]) resultList.get(i).get("V"))); + } + + // query with empty scan range + tableQuery = obTableClient.query("testRange"); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + result = tableQuery.execute(); + Assert.assertEquals(4, result.cacheSize()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", new Object[] { "a1", "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete("testRange", new Object[] { "a1", "partition".getBytes(), timeStamp + 2 }); + obTableClient.delete("testRange", new Object[] { "a1", "partition".getBytes(), timeStamp + 3 }); + obTableClient.delete("testRange", new Object[] { "x1", "partition".getBytes(), timeStamp + 1 }); + } + } + + @Test + public void testAsyncQueryLocalIndex() throws Exception { + // TODO: client's route with index is wrong + if (!obTableClient.isOdpMode()) { + return; + } + long timeStamp = System.currentTimeMillis(); + try { + // the client's route sucks, cannot work in non-odp mode currently + // p1 + obTableClient.insert("testRange", + new Object[] { "a1", "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testRange", + new Object[] { "a1", "partition".getBytes(), timeStamp + 2}, + new String[] { "V" }, new Object[] { "value3".getBytes() }); + obTableClient.insert("testRange", + new Object[] { "a1", "partition".getBytes(), timeStamp + 3}, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + // p2 + obTableClient.insert("testRange", + new Object[] { "x1", "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + + // single partition query with index + TableQuery tableQuery = obTableClient.query("testRange"); + tableQuery.addScanRange(new Object[] { "a1", "value1".getBytes() }, + new Object[] { "a1", "value9".getBytes() }); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(2); + QueryResultSet result = tableQuery.asyncExecute(); + + int[] tsDelta = {1, 3, 2}; + for (int i = 1; i <= 3; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("a1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + tsDelta[i - 1], row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // query with index backward + tableQuery.scanOrder(false); + result = tableQuery.asyncExecute(); + Assert.assertEquals(3, result.cacheSize()); + for (int i = 3; i >= 1; i--) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals("a1", new String((byte[]) row.get("K"))); + Assert.assertEquals("partition", new String((byte[]) row.get("Q"))); + Assert.assertEquals(timeStamp + tsDelta[i - 1], row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // single partition query with index prefix + tableQuery = obTableClient.query("testRange"); + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[] { "a1" }, + new Object[] { "a1" }); + tableQuery.select("K", "T", "V"); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(2); + result = tableQuery.asyncExecute(); + + for (int i = 1; i <= 3; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + Assert.assertEquals(3, row.size()); + Assert.assertEquals("a1", new String((byte[]) row.get("K"))); + Assert.assertEquals(timeStamp + tsDelta[i - 1], row.get("T")); + Assert.assertEquals("value" + i, new String((byte[]) row.get("V"))); + } + Assert.assertFalse(result.next()); + + // multiply partition query with index + tableQuery = obTableClient.query("testRange"); + tableQuery.addScanRange(new Object[] { "a0", "value1" }, new Object[] { "z9", "value9" } ); + tableQuery.select("K", "V", "Q", "T"); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(2); + result = tableQuery.asyncExecute(); + Assert.assertEquals(4, result.cacheSize()); + + // sort result by K, T + List> resultList = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + Assert.assertTrue(result.next()); + Assert.assertEquals(4, result.getRow().size()); + resultList.add(result.getRow()); + } + Assert.assertFalse(result.next()); + Collections.sort(resultList, new Comparator>() { + @Override + public int compare(Map o1, Map o2) { + int firstCmp = new String((byte[])o1.get("K")).compareTo(new String((byte[])o2.get("K"))); + if (firstCmp == 0) { + return (int)(((long)o1.get("T")) - ((long)o2.get("T"))); + } + return firstCmp; + } + }); + String[] orderedKeys = {"a1", "a1", "a1", "x1"}; + int[] orderedDeltaTs = {1, 2, 3, 1}; + String[] orderedValues = {"value1", "value3", "value2", "value1"}; + for (int i = 0; i < 4; i++) { + Assert.assertEquals(orderedKeys[i], new String((byte[])resultList.get(i).get("K"))); + Assert.assertEquals("partition", new String((byte[]) resultList.get(i).get("Q"))); + Assert.assertEquals(timeStamp + orderedDeltaTs[i], resultList.get(i).get("T")); + Assert.assertEquals(orderedValues[i], new String((byte[]) resultList.get(i).get("V"))); + } + + // query with empty scan range + tableQuery = obTableClient.query("testRange"); + tableQuery.setScanRangeColumns("K", "V"); + tableQuery.indexName("i1"); + tableQuery.setBatchSize(2); + result = tableQuery.asyncExecute(); + Assert.assertEquals(4, result.cacheSize()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", new Object[] { "a1", "partition".getBytes(), timeStamp + 1 }); + obTableClient.delete("testRange", new Object[] { "a1", "partition".getBytes(), timeStamp + 2 }); + obTableClient.delete("testRange", new Object[] { "a1", "partition".getBytes(), timeStamp + 3 }); + obTableClient.delete("testRange", new Object[] { "x1", "partition".getBytes(), timeStamp + 1 }); + } } @Test public void testBatch() throws Exception { long timeStamp = System.currentTimeMillis(); - obTableClient.insert("testRange", new Object[] { "ah", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - obTableClient.insert("testRange", new Object[] { "xh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - TableBatchOps tableBatchOps = obTableClient.batch("testRange"); - tableBatchOps.delete(new Object[] { "ah", "partition".getBytes(), timeStamp }); - tableBatchOps.insert(new Object[] { "hh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - tableBatchOps.replace(new Object[] { "xh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - List batchResult = tableBatchOps.execute(); - Assert.assertEquals(3, batchResult.size()); - Assert.assertEquals(1L, batchResult.get(0)); - Assert.assertEquals(1L, batchResult.get(1)); - Assert.assertEquals(2L, batchResult.get(2)); - - Map getResult = obTableClient.get("testRange", new Object[] { "ah", - "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); - - Assert.assertEquals(0, getResult.size()); - - getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), - timeStamp }, new String[] { "K", "Q", "T", "V" }); - - Assert.assertEquals(4, getResult.size()); - - Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); - - getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), - timeStamp }, new String[] { "K", "Q", "T", "V" }); - - Assert.assertEquals(4, getResult.size()); - - Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + try { + obTableClient.insert("testRange", new Object[] { "ah", "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value1".getBytes() }); + TableBatchOps tableBatchOps = obTableClient.batch("testRange"); + tableBatchOps.delete(new Object[] { "ah", "partition".getBytes(), timeStamp }); + tableBatchOps.insert(new Object[] { "hh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + tableBatchOps.replace(new Object[] { "xh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + List batchResult = tableBatchOps.execute(); + Assert.assertEquals(3, batchResult.size()); + Assert.assertEquals(1L, batchResult.get(0)); + Assert.assertEquals(1L, batchResult.get(1)); + Assert.assertEquals(2L, batchResult.get(2)); + + Map getResult = obTableClient.get("testRange", new Object[] { "ah", + "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(0, getResult.size()); + + getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }, new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(4, getResult.size()); + + Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + + getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }, new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(4, getResult.size()); + + Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", new Object[] { "ah", "partition".getBytes(), + timeStamp }); + obTableClient.delete("testRange", new Object[] { "hh", "partition".getBytes(), + timeStamp }); + obTableClient.delete("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }); + } } @Test public void testBatchConcurrent() throws Exception { long timeStamp = System.currentTimeMillis(); obTableClient.setRuntimeBatchExecutor(Executors.newFixedThreadPool(3)); - obTableClient.insert("testRange", new Object[] { "ah", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - obTableClient.insert("testRange", new Object[] { "xh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - TableBatchOps tableBatchOps = obTableClient.batch("testRange"); - tableBatchOps.delete(new Object[] { "ah", "partition".getBytes(), timeStamp }); - tableBatchOps.insert(new Object[] { "hh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - tableBatchOps.replace(new Object[] { "xh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - List batchResult = tableBatchOps.execute(); - Assert.assertEquals(3, batchResult.size()); - Assert.assertEquals(1L, batchResult.get(0)); - Assert.assertEquals(1L, batchResult.get(1)); - Assert.assertEquals(2L, batchResult.get(2)); - - Map getResult = obTableClient.get("testRange", new Object[] { "ah", - "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); - - Assert.assertEquals(0, getResult.size()); - - getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), - timeStamp }, new String[] { "K", "Q", "T", "V" }); - - Assert.assertEquals(4, getResult.size()); - - Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); - - getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), - timeStamp }, new String[] { "K", "Q", "T", "V" }); - - Assert.assertEquals(4, getResult.size()); - - Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + try { + obTableClient.insert("testRange", new Object[] { "ah", "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value1".getBytes() }); + TableBatchOps tableBatchOps = obTableClient.batch("testRange"); + tableBatchOps.delete(new Object[] { "ah", "partition".getBytes(), timeStamp }); + tableBatchOps.insert(new Object[] { "hh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + tableBatchOps.replace(new Object[] { "xh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + List batchResult = tableBatchOps.execute(); + Assert.assertEquals(3, batchResult.size()); + Assert.assertEquals(1L, batchResult.get(0)); + Assert.assertEquals(1L, batchResult.get(1)); + Assert.assertEquals(2L, batchResult.get(2)); + + Map getResult = obTableClient.get("testRange", new Object[] { "ah", + "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(0, getResult.size()); + + getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }, new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(4, getResult.size()); + + Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + + getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }, new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(4, getResult.size()); + + Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", new Object[] { "ah", "partition".getBytes(), + timeStamp }); + obTableClient.delete("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }); + obTableClient.delete("testRange", new Object[] { "hh", "partition".getBytes(), + timeStamp }); + } } @Test public void testBatchConcurrentWithPriority() throws Exception { long timeStamp = System.currentTimeMillis(); ThreadLocalMap.setProcessHighPriority(); - obTableClient.setRuntimeBatchExecutor(Executors.newFixedThreadPool(3)); - obTableClient.insert("testRange", new Object[] { "ah", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - obTableClient.insert("testRange", new Object[] { "xh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value1".getBytes() }); - TableBatchOps tableBatchOps = obTableClient.batch("testRange"); - tableBatchOps.delete(new Object[] { "ah", "partition".getBytes(), timeStamp }); - tableBatchOps.insert(new Object[] { "hh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - tableBatchOps.replace(new Object[] { "xh", "partition".getBytes(), timeStamp }, - new String[] { "V" }, new Object[] { "value2".getBytes() }); - List batchResult = tableBatchOps.execute(); - Assert.assertEquals(3, batchResult.size()); - Assert.assertEquals(1L, batchResult.get(0)); - Assert.assertEquals(1L, batchResult.get(1)); - Assert.assertEquals(2L, batchResult.get(2)); - - Map getResult = obTableClient.get("testRange", new Object[] { "ah", - "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); - - Assert.assertEquals(0, getResult.size()); - - getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), - timeStamp }, new String[] { "K", "Q", "T", "V" }); - - Assert.assertEquals(4, getResult.size()); - - Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); - - getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), - timeStamp }, new String[] { "K", "Q", "T", "V" }); - - Assert.assertEquals(4, getResult.size()); - - Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); - Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); - Assert.assertEquals(timeStamp, getResult.get("T")); - Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + try { + obTableClient.setRuntimeBatchExecutor(Executors.newFixedThreadPool(3)); + obTableClient.insert("testRange", new Object[] { "ah", "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value1".getBytes() }); + obTableClient.insert("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }, new String[] { "V" }, new Object[] { "value1".getBytes() }); + TableBatchOps tableBatchOps = obTableClient.batch("testRange"); + tableBatchOps.delete(new Object[] { "ah", "partition".getBytes(), timeStamp }); + tableBatchOps.insert(new Object[] { "hh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + tableBatchOps.replace(new Object[] { "xh", "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + List batchResult = tableBatchOps.execute(); + Assert.assertEquals(3, batchResult.size()); + Assert.assertEquals(1L, batchResult.get(0)); + Assert.assertEquals(1L, batchResult.get(1)); + Assert.assertEquals(2L, batchResult.get(2)); + + Map getResult = obTableClient.get("testRange", new Object[] { "ah", + "partition".getBytes(), timeStamp }, new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(0, getResult.size()); + + getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }, new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(4, getResult.size()); + + Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + + getResult = obTableClient.get("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }, new String[] { "K", "Q", "T", "V" }); + + Assert.assertEquals(4, getResult.size()); + + Assert.assertEquals("xh", new String((byte[]) getResult.get("K"))); + Assert.assertEquals("partition", new String((byte[]) getResult.get("Q"))); + Assert.assertEquals(timeStamp, getResult.get("T")); + Assert.assertEquals("value2", new String((byte[]) getResult.get("V"))); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + obTableClient.delete("testRange", new Object[] { "ah", "partition".getBytes(), + timeStamp }); + obTableClient.delete("testRange", new Object[] { "xh", "partition".getBytes(), + timeStamp }); + obTableClient.delete("testRange", new Object[] { "hh", "partition".getBytes(), + timeStamp }); + } } + @Test + public void testPartitionLocation() throws Exception { + obTableClient.setRunningMode(ObTableClient.RunningMode.NORMAL); + String testTable = "testPartitionRangeComplex"; + obTableClient.addRowKeyElement(testTable, new String[] { "c1", "c2", "c3", "c4" }); + Random rng = new Random(); + try { + cleanTable(testTable); + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + for (int i = 0; i < 64; i++) { + int c1 = abs(rng.nextInt()) % 2000; + long c2 = abs(rng.nextLong()) % 2000; + String c3 = generateRandomStringByUUID(10); + String c4 = generateRandomStringByUUID(5) + c3 + generateRandomStringByUUID(5); + + // use sql to insert data + statement.execute("insert into " + testTable + "(c1, c2, c3, c4, c5) values (" + c1 + + "," + c2 + ",'" + c3 + "','" + c4 + "'," + "'value')"); + + // get data by obkv interface + Map result = obTableClient.get(testTable, + new Object[] { c1, c2, c3.getBytes(), c4 }, new String[] { "c1", "c2", "c3", + "c4", "c5" }); + Assert.assertEquals(5, result.size()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanTable(testTable); + } + } + + @Test + public void testPartitionLocationDateTime() throws Exception { + // This test case may wrong when you are not in GMT+8 time zone + obTableClient.setRunningMode(ObTableClient.RunningMode.NORMAL); + String testTable = "testDateTime"; + obTableClient.addRowKeyElement(testTable, new String[] { "c0", "c1" }); + try { + cleanTable(testTable); + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + for (int i = 0; i < 64; i++) { + // 1650340800000 -> 2022-04-19 15:00:00 (GMT +8) + SimpleDateFormat sdf = new SimpleDateFormat(" yyyy-MM-dd HH:mm:ss "); + Date date = new Date(1650340800000L + 10800000L * i); + String formattedDate = sdf.format(date); + + // use sql to insert data + statement.execute("insert into " + testTable + " values ( '" + formattedDate + + "' , '" + formattedDate + "' ," + "'value')"); + + // get data by obkv interface + Map result = obTableClient.get(testTable, + new Object[] { date, date }, new String[] { "c0", "c1", "c2" }); + Assert.assertEquals(3, result.size()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanTable(testTable); + } + } + + @Test + public void testNotInAnyPartition() throws Exception { + obTableClient.setRunningMode(ObTableClient.RunningMode.NORMAL); + String testTable = "testDateTime"; + obTableClient.addRowKeyElement(testTable, new String[] { "c0", "c1" }); + try { + cleanTable(testTable); + + // 1651334400000L -> 2022-05-01 00:00:00 (GMT +8) + Date date = new Date(1651334400000L); + long affectedRows = obTableClient.insert(testTable, new Object[] { date, date }, + new String[] { "c2" }, new Object[] { "value" }); + Assert.assertTrue("Insert should fail since no partition could be inserted", false); + } catch (IndexOutOfBoundsException e) { + // do nothing + Assert.assertEquals("Table has no partition for value in c0", e.getMessage()); + } catch (Exception e) { + Assert.assertTrue("Unexpected exception has occurred", false); + } finally { + cleanTable(testTable); + } + } + + @Test + public void testFirstPartStartEndKeys() throws Exception { + // CREATE TABLEGROUP test_start_end_keys_range SHARDING = 'ADAPTIVE'; + // CREATE TABLE `test_start_end_keys_range$family_range` ( + // `K` varbinary(1024) NOT NULL, + // `Q` varbinary(256) NOT NULL, + // `T` bigint(20) NOT NULL, + // `V` varbinary(1024) DEFAULT NULL, + // PRIMARY KEY (`K`, `Q`, `T`) + // ) partition by range columns (`K`) ( + // PARTITION p0 VALUES LESS THAN ('a'), + // PARTITION p1 VALUES LESS THAN ('w'), + // PARTITION p2 VALUES LESS THAN MAXVALUE + // ) TABLEGROUP = test_start_end_keys_range; + + try { + byte[][][] rangeFirstPartStartKeys = obTableClient + .getFirstPartStartKeys("test_start_end_keys_range$family_range"); + byte[][][] rangeFirstPartEndKeys = obTableClient + .getFirstPartEndKeys("test_start_end_keys_range$family_range"); + int keySize = rangeFirstPartStartKeys.length; + Assert + .assertArrayEquals(rangeFirstPartStartKeys[0], rangeFirstPartEndKeys[keySize - 1]); + for (int i = 1; i < keySize; ++i) { + Assert.assertArrayEquals(rangeFirstPartStartKeys[i], rangeFirstPartEndKeys[i - 1]); + } + } catch (Exception e) { + fail(); + } + } + + @Test + public void testHBaseStartEndKeys() throws Exception { + // CREATE TABLEGROUP test_start_end_keys_range SHARDING = 'ADAPTIVE'; + // CREATE TABLE `test_start_end_keys_range$family_range` ( + // `K` varbinary(1024) NOT NULL, + // `Q` varbinary(256) NOT NULL, + // `T` bigint(20) NOT NULL, + // `V` varbinary(1024) DEFAULT NULL, + // PRIMARY KEY (`K`, `Q`, `T`) + // ) partition by range columns (`K`) ( + // PARTITION p0 VALUES LESS THAN ('a'), + // PARTITION p1 VALUES LESS THAN ('w'), + // PARTITION p2 VALUES LESS THAN MAXVALUE + // ) TABLEGROUP = test_start_end_keys_range; + + try { + byte[][] rangeHBaseStartKeys = obTableClient + .getHBaseTableStartKeys("test_start_end_keys_range"); + byte[][] rangeHBaseEndKeys = obTableClient + .getHBaseTableEndKeys("test_start_end_keys_range"); + int keySize = rangeHBaseStartKeys.length; + Assert.assertArrayEquals(rangeHBaseStartKeys[0], rangeHBaseEndKeys[keySize - 1]); + for (int i = 1; i < keySize; ++i) { + Assert.assertArrayEquals(rangeHBaseStartKeys[i], rangeHBaseEndKeys[i - 1]); + } + } catch (Exception e) { + fail(); + } + } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientTest.java index ff4d5571..1d4e2b43 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableClientTest.java @@ -18,24 +18,49 @@ package com.alipay.oceanbase.rpc; import com.alipay.oceanbase.rpc.bolt.ObTableClientTestBase; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.exception.ObTablePartitionConsistentException; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.filter.*; import com.alipay.oceanbase.rpc.location.model.ObServerAddr; import com.alipay.oceanbase.rpc.location.model.ServerRoster; +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.mutation.result.*; import com.alipay.oceanbase.rpc.property.Property; +import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateRequest; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateResult; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; import com.alipay.oceanbase.rpc.table.api.Table; import com.alipay.oceanbase.rpc.table.api.TableBatchOps; +import com.alipay.oceanbase.rpc.table.api.TableQuery; import com.alipay.oceanbase.rpc.threadlocal.ThreadLocalMap; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import com.alipay.oceanbase.rpc.util.ObTableHotkeyThrottleUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.*; +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.*; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.*; +import static com.alipay.oceanbase.rpc.util.ObTableClientTestUtil.cleanTable; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class ObTableClientTest extends ObTableClientTestBase { @Before @@ -44,12 +69,17 @@ public void setup() throws Exception { final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); obTableClient.setMetadataRefreshInterval(100); - obTableClient.addProperty("connectTimeout", "100000"); - obTableClient.addProperty("socketTimeout", "100000"); - obTableClient.addProperty("table.connection.pool.size", "3"); + obTableClient.addProperty(Property.RPC_CONNECT_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.RPC_LOGIN_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.SERVER_CONNECTION_POOL_SIZE.getKey(), "1"); + obTableClient.addProperty(Property.RPC_EXECUTE_TIMEOUT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_MAX_WAIT.getKey(), "3000"); + obTableClient.addProperty(Property.RUNTIME_BATCH_EXECUTOR.getKey(), "32"); + obTableClient.addProperty(Property.RPC_OPERATION_TIMEOUT.getKey(), "3000"); + obTableClient.addProperty(Property.SERVER_ENABLE_REROUTING.getKey(), "False"); obTableClient.init(); - client = obTableClient; + this.client = obTableClient; syncRefreshMetaHelper(obTableClient); } @@ -68,18 +98,20 @@ private long getMaxAccessTime(ObTableClient client) throws Exception { @Test public void testMetadataRefresh() throws Exception { final ObTableClient client1 = ObTableClientTestUtil.newTestClient(); - try { - client1.setMetadataRefreshInterval(100); - client1.setServerAddressCachingTimeout(8000); - client1.init(); - long lastTime = getMaxAccessTime(client1); - Thread.sleep(10000); - client1.insertOrUpdate("test_varchar_table", "foo", new String[] { "c2" }, - new String[] { "bar" }); - long nowTime = getMaxAccessTime(client1); - Assert.assertTrue(nowTime - lastTime > 8000); - } finally { - client1.delete("test_varchar_table", "foo"); + if (!client1.isOdpMode()) { + try { + client1.setMetadataRefreshInterval(100); + client1.setServerAddressCachingTimeout(8000); + client1.init(); + long lastTime = getMaxAccessTime(client1); + Thread.sleep(10000); + client1.insertOrUpdate("test_varchar_table", "foo", new String[] { "c2" }, + new String[] { "bar" }); + long nowTime = getMaxAccessTime(client1); + Assert.assertTrue(nowTime - lastTime > 8000); + } finally { + client1.delete("test_varchar_table", "foo"); + } } } @@ -125,7 +157,6 @@ public void testPropertiesException() throws Exception { ObTableClient obTableClient1 = ObTableClientTestUtil.newTestClient(); obTableClient1.addProperty("connectTimeout", "100000"); obTableClient1.addProperty("socketTimeout", "100000"); - obTableClient1.setRuntimeRetryTimes(-1); obTableClient1.init(); Assert.assertEquals(obTableClient1.getRuntimeRetryTimes(), @@ -135,10 +166,10 @@ public void testPropertiesException() throws Exception { ObTableClient obTableClient2 = ObTableClientTestUtil.newTestClient(); obTableClient2.addProperty("connectTimeout", "100000"); obTableClient2.addProperty("socketTimeout", "100000"); - obTableClient2.addProperty(Property.RUNTIME_RETRY_TIMES.getKey(), "-1"); + obTableClient2.addProperty(Property.RUNTIME_RETRY_TIMES.getKey(), "0"); obTableClient2.init(); - Assert.assertEquals(obTableClient2.getRuntimeRetryTimes(), 1); + Assert.assertEquals(obTableClient2.getRuntimeRetryTimes(), 0); obTableClient2.close(); } @@ -467,4 +498,2203 @@ public void testAddrExpired() throws Exception { } } + @Test + public void testQueryWithFilter() throws Exception { + + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + + try { + client.insert("test_query_filter_mutate", new Object[] { 0L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row1" }); + client.insert("test_query_filter_mutate", new Object[] { 1L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row2" }); + client.insert("test_query_filter_mutate", new Object[] { 2L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row3" }); + + TableQuery tableQuery = client.query("test_query_filter_mutate"); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 250L }); + tableQuery.select("c1", "c2", "c3"); + + ObTableValueFilter filter_0 = new ObTableValueFilter(ObCompareOp.GT, "c1", 0); + ObTableValueFilter filter_1 = new ObTableValueFilter(ObCompareOp.LE, "c1", 2); + ObTableValueFilter filter_2 = new ObTableValueFilter(ObCompareOp.GT, "c1", 1); + + tableQuery.setFilter(filter_0); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + tableQuery.setFilter(filter_1); + result = tableQuery.execute(); + Assert.assertEquals(3, result.cacheSize()); + + tableQuery.setFilter(filter_2); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + } + } + + @Test + public void testAsyncQueryWithFilter() throws Exception { + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + + try { + client.insert("test_query_filter_mutate", new Object[] { 0L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row1" }); + client.insert("test_query_filter_mutate", new Object[] { 1L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row2" }); + client.insert("test_query_filter_mutate", new Object[] { 2L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row3" }); + + TableQuery tableQuery = client.query("test_query_filter_mutate"); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 250L }); + tableQuery.select("c1", "c2", "c3"); + tableQuery.setBatchSize(1); + + ObTableValueFilter filter_0 = new ObTableValueFilter(ObCompareOp.GT, "c1", 0); + ObTableValueFilter filter_1 = new ObTableValueFilter(ObCompareOp.LE, "c1", 2); + ObTableValueFilter filter_2 = new ObTableValueFilter(ObCompareOp.GT, "c1", 1); + + tableQuery.setFilter(filter_0); + QueryResultSet result = tableQuery.asyncExecute(); + int expected_ret = 2; + while (result.next()) { + expected_ret -= 1; + } + Assert.assertEquals(0, expected_ret); + + tableQuery.setFilter(filter_1); + result = tableQuery.asyncExecute(); + expected_ret = 3; + while (result.next()) { + expected_ret -= 1; + } + Assert.assertEquals(0, expected_ret); + + tableQuery.setFilter(filter_2); + result = tableQuery.asyncExecute(); + expected_ret = 1; + while (result.next()) { + expected_ret -= 1; + } + Assert.assertEquals(0, expected_ret); + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + } + } + + @Test + public void testQueryAndAppend() throws Exception { + + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + + try { + client.insert("test_query_filter_mutate", new Object[] { 0L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row1" }); + client.insert("test_query_filter_mutate", new Object[] { 1L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row2" }); + client.insert("test_query_filter_mutate", new Object[] { 2L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row3" }); + TableQuery tableQuery = client.query("test_query_filter_mutate"); + /* Scan range must in one partition */ + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.select("c1", "c2", "c3"); + + /* Set Filter String */ + ObTableValueFilter filter_0 = compareVal(ObCompareOp.GT, "c1", 0); + ObTableValueFilter filter_1 = compareVal(ObCompareOp.GE, "c3", "row3_append0"); + + tableQuery.setFilter(filter_0); + ObTableQueryAndMutateRequest request_0 = ((ObTableClient) client) + .obTableQueryAndAppend(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "_append0" }, true); + ObPayload res_exec_0 = ((ObTableClient) client).execute(request_0); + ObTableQueryAndMutateResult res = (ObTableQueryAndMutateResult) res_exec_0; + Assert.assertEquals(2, res.getAffectedRows()); + /* check value before append */ + Assert.assertEquals("row2", res.getAffectedEntity().getPropertiesRows().get(0).get(2) + .getValue()); + /* To confirm changing. re-query to get the latest data */; + ObTableValueFilter confirm_0 = compareVal(ObCompareOp.EQ, "c3", "row2_append0"); + tableQuery.setFilter(confirm_0); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + + tableQuery.setFilter(filter_1); + ObTableQueryAndMutateRequest request_1 = ((ObTableClient) client) + .obTableQueryAndAppend(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "_append1" }, true); + ObPayload res_exec_1 = ((ObTableClient) client).execute(request_1); + res = (ObTableQueryAndMutateResult) res_exec_1; + Assert.assertEquals(1, res.getAffectedRows()); + ObTableValueFilter confirm_1 = compareVal(ObCompareOp.EQ, "c3", "row3_append0_append1"); + tableQuery.setFilter(confirm_1); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + } + } + + @Test + public void testQueryAndIncrement() throws Exception { + + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + + try { + client.insert("test_query_filter_mutate", new Object[] { 0L }, new String[] { "c2", + "c3", "c4" }, new Object[] { new byte[] { 1 }, "row1", 0L }); + client.insert("test_query_filter_mutate", new Object[] { 1L }, new String[] { "c2", + "c3", "c4" }, new Object[] { new byte[] { 1 }, "row2", 10L }); + client.insert("test_query_filter_mutate", new Object[] { 2L }, new String[] { "c2", + "c3", "c4" }, new Object[] { new byte[] { 1 }, "row3", 20L }); + + TableQuery tableQuery = client.query("test_query_filter_mutate"); + /* Scan range must in one partition */ + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.select("c1", "c2", "c3", "c4"); + + /* Set Filter String */ + ObTableValueFilter filter_0 = compareVal(ObCompareOp.GT, "c1", 0); + ObTableValueFilter filter_1 = compareVal(ObCompareOp.LT, "c3", "row3"); + + try { + ObTableQueryAndMutateRequest request = ((ObTableClient) client) + .obTableQueryAndIncrement(tableQuery, null, null, true); + ObPayload res_exec = ((ObTableClient) client).execute(request); + } catch (Exception e) { + assertTrue(true); + } + tableQuery.setFilter(filter_0); + ObTableQueryAndMutateRequest request_0 = ((ObTableClient) client) + .obTableQueryAndIncrement(tableQuery, new String[] { "c4" }, new Object[] { 5L }, + true); + ObPayload res_exec_0 = ((ObTableClient) client).execute(request_0); + ObTableQueryAndMutateResult res = (ObTableQueryAndMutateResult) res_exec_0; + Assert.assertEquals(2, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_0 = compareVal(ObCompareOp.GE, "c4", 15); + tableQuery.setFilter(confirm_0); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + tableQuery.setFilter(filter_1); + ObTableQueryAndMutateRequest request_1 = ((ObTableClient) client) + .obTableQueryAndIncrement(tableQuery, new String[] { "c4" }, new Object[] { 7L }, + true); + ObPayload res_exec_1 = ((ObTableClient) client).execute(request_1); + res = (ObTableQueryAndMutateResult) res_exec_1; + Assert.assertEquals(2, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_1 = compareVal(ObCompareOp.EQ, "c4", 22); + tableQuery.setFilter(confirm_1); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + } + } + + @Test + public void testQueryAndDelete() throws Exception { + + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + + try { + client.insert("test_query_filter_mutate", new Object[] { 0L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row1" }); + client.insert("test_query_filter_mutate", new Object[] { 1L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row2" }); + client.insert("test_query_filter_mutate", new Object[] { 2L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row3" }); + + TableQuery tableQuery = client.query("test_query_filter_mutate"); + /* Scan range must in one partition */ + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.select("c1", "c2", "c3"); + + /* Set Filter String */ + ObTableValueFilter filter_0 = compareVal(ObCompareOp.EQ, "c1", 0); + ObTableValueFilter filter_1 = new ObTableValueFilter(ObCompareOp.GT, "c3", "ro"); + ObTableValueFilter filter_2 = new ObTableValueFilter(ObCompareOp.LT, "c3", "row3"); + ObTableFilterList filterList = andList(filter_1, filter_2); + ObTableValueFilter filter_3 = new ObTableValueFilter(ObCompareOp.LT, null, "row3"); + + tableQuery.setFilter(filter_0); + ObTableQueryAndMutateRequest request_0 = ((ObTableClient) client) + .obTableQueryAndDelete(tableQuery); + ObPayload res_exec_0 = ((ObTableClient) client).execute(request_0); + ObTableQueryAndMutateResult res = (ObTableQueryAndMutateResult) res_exec_0; + Assert.assertEquals(1, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_0 = compareVal(ObCompareOp.GE, "c1", 0); + tableQuery.setFilter(confirm_0); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + tableQuery.setFilter(filterList); + ObTableQueryAndMutateRequest request_1 = ((ObTableClient) client) + .obTableQueryAndDelete(tableQuery); + ObPayload res_exec_1 = ((ObTableClient) client).execute(request_1); + res = (ObTableQueryAndMutateResult) res_exec_1; + Assert.assertEquals(1, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_1 = compareVal(ObCompareOp.GE, "c1", 0); + tableQuery.setFilter(confirm_1); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + + tableQuery.setFilter(filter_3); + ObTableQueryAndMutateRequest request_2 = ((ObTableClient) client) + .obTableQueryAndDelete(tableQuery); + ObPayload res_exec_2 = ((ObTableClient) client).execute(request_2); + res = (ObTableQueryAndMutateResult) res_exec_2; + Assert.assertEquals(1, res.getAffectedRows()); + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + } + } + + @Test + public void testQueryAndUpdate() throws Exception { + + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + + try { + client.insert("test_query_filter_mutate", new Object[] { 0L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row1" }); + client.insert("test_query_filter_mutate", new Object[] { 1L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row2" }); + client.insert("test_query_filter_mutate", new Object[] { 2L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row3" }); + + TableQuery tableQuery = client.query("test_query_filter_mutate"); + /* Scan range must in one partition */ + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.select("c1", "c2", "c3"); + + ObTableValueFilter filter_0 = compareVal(ObCompareOp.GT, "c1", 0); + ObTableValueFilter filter_1 = compareVal(ObCompareOp.EQ, "c3", "update1"); + + try { + ObTableQueryAndMutateRequest request = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, null, null); + ObPayload res_exec = ((ObTableClient) client).execute(request); + } catch (Exception e) { + assertTrue(true); + } + + tableQuery.setFilter(filter_0); + ObTableQueryAndMutateRequest request_0 = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "update1" }); + ObPayload res_exec_0 = ((ObTableClient) client).execute(request_0); + ObTableQueryAndMutateResult res = (ObTableQueryAndMutateResult) res_exec_0; + Assert.assertEquals(2, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_0 = compareVal(ObCompareOp.EQ, "c3", "update1"); + tableQuery.setFilter(confirm_0); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + tableQuery.setFilter(filter_1); + ObTableQueryAndMutateRequest request_1 = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "update2" }); + ObPayload res_exec_1 = ((ObTableClient) client).execute(request_1); + res = (ObTableQueryAndMutateResult) res_exec_1; + Assert.assertEquals(2, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_1 = compareVal(ObCompareOp.EQ, "c3", "update2"); + tableQuery.setFilter(confirm_1); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + if (client instanceof ObTableClient && ((ObTableClient) client).isOdpMode()) { + try { + tableQuery = client.query("test_query_filter_mutate"); + tableQuery.select("c1", "c2", "c3"); + tableQuery.setFilter(filter_0); + ObTableQueryAndMutateRequest request_2 = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, new String[] { "c2", "c3" }, + new Object[] { new byte[] { 1 }, "update1" }); + ObPayload res_exec_2 = ((ObTableClient) client).execute(request_2); + Assert.assertTrue(false); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableException); + if (client instanceof ObTableClient && ((ObTableClient) client).isOdpMode()) { + Assert.assertEquals(ResultCodes.OB_ERR_UNEXPECTED.errorCode, + ((ObTableUnexpectedException) e).getErrorCode()); + } else { + Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, + ((ObTableException) e).getErrorCode()); + } + } + + } + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + } + } + + @Test + public void testQueryAndMutateComplex() throws Exception { + + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + + try { + client.insert("test_query_filter_mutate", new Object[] { 0L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row1" }); + client.insert("test_query_filter_mutate", new Object[] { 1L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row2" }); + client.insert("test_query_filter_mutate", new Object[] { 2L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row3" }); + client.insert("test_query_filter_mutate", new Object[] { 3L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row4" }); + client.insert("test_query_filter_mutate", new Object[] { 4L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row5" }); + client.insert("test_query_filter_mutate", new Object[] { 5L }, new String[] { "c2", + "c3" }, new Object[] { new byte[] { 1 }, "row6" }); + client.insert("test_query_filter_mutate").setRowKey(new Row("c1", 10L)) + .addMutateColVal(new ColumnValue("c2", new byte[] { 1 })) + .addMutateColVal(new ColumnValue("c3", "z_row")) + .addMutateColVal(new ColumnValue("c4", 10L)).execute(); + + TableQuery tableQuery = client.query("test_query_filter_mutate"); + /* Scan range must in one partition */ + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.select("c1", "c2", "c3"); + + ObTableValueFilter c1_GT_0 = compareVal(ObCompareOp.GT, "c1", 0); + ObTableValueFilter c1_EQ_0 = compareVal(ObCompareOp.EQ, "c1", 0); + ObTableValueFilter c1_LE_0 = compareVal(ObCompareOp.LE, "c1", 0); + ObTableValueFilter c1_LT_5 = compareVal(ObCompareOp.LT, "c1", 5); + ObTableValueFilter c1_LE_5 = compareVal(ObCompareOp.LE, "c1", 5); + ObTableValueFilter c1_GT_3 = compareVal(ObCompareOp.GT, "c1", 3); + ObTableValueFilter c1_LT_2 = compareVal(ObCompareOp.LT, "c1", 2); + ObTableValueFilter c1_EQ_5 = compareVal(ObCompareOp.EQ, "c1", 5); + ObTableValueFilter c3_GE = compareVal(ObCompareOp.GE, "c3", "update"); + ObTableValueFilter c3_LT = compareVal(ObCompareOp.LT, "c3", "update4"); + ObTableFilterList filters_0 = andList(); + ObTableFilterList filters_1 = andList(); + ObTableFilterList filters_2 = orList(); + + // c1 = 0 && c1 = 0 + filters_0.addFilter(c1_EQ_0, c1_EQ_0); + tableQuery.setFilter(filters_0); + ObTableQueryAndMutateRequest request_0 = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "update1" }); + ObPayload res_exec_0 = ((ObTableClient) client).execute(request_0); + ObTableQueryAndMutateResult res = (ObTableQueryAndMutateResult) res_exec_0; + Assert.assertEquals(1, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_0 = compareVal(ObCompareOp.EQ, "c3", "update1"); + tableQuery.setFilter(confirm_0); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + + // c1 = 0 && (c1 = 0 && c1 = 0) + filters_1.addFilter(c1_EQ_0, filters_0); + tableQuery.setFilter(filters_1); + ObTableQueryAndMutateRequest request_1 = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "update2" }); + ObPayload res_exec_1 = ((ObTableClient) client).execute(request_1); + res = (ObTableQueryAndMutateResult) res_exec_1; + Assert.assertEquals(1, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_1 = compareVal(ObCompareOp.EQ, "c3", "update2"); + tableQuery.setFilter(confirm_1); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + + // c1 = 5 || (c1 > 3 && c1 <= 5) + filters_0 = andList(c1_GT_3, c1_LE_5); + filters_1 = orList(c1_EQ_5, filters_0); + tableQuery.setFilter(filters_1); + ObTableQueryAndMutateRequest request_2 = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "update3" }); + ObPayload res_exec_2 = ((ObTableClient) client).execute(request_2); + res = (ObTableQueryAndMutateResult) res_exec_2; + Assert.assertEquals(2, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_2 = compareVal(ObCompareOp.EQ, "c3", "update3"); + tableQuery.setFilter(confirm_2); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + // (c1 > 0 && c1 < 5) || (c1 <= 0 || c1 < 2) + filters_0 = andList(c1_GT_0, c1_LT_5); + filters_1 = orList(c1_LE_0, c1_LT_2); + filters_2.addFilter(filters_0, filters_1); + tableQuery.setFilter(filters_2); + ObTableQueryAndMutateRequest request_3 = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "update4" }); + ObPayload res_exec_3 = ((ObTableClient) client).execute(request_3); + res = (ObTableQueryAndMutateResult) res_exec_3; + Assert.assertEquals(5, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_3 = compareVal(ObCompareOp.EQ, "c3", "update4"); + tableQuery.setFilter(confirm_3); + result = tableQuery.execute(); + Assert.assertEquals(5, result.cacheSize()); + + // (c3 >= update && c3 < update4 && c1 < 2) || (c3 < update4 && c1 > 3) + filters_0 = andList(c3_GE, c3_LT, c1_LT_2); + filters_1 = andList(c3_LT, c1_GT_3); + filters_2 = orList(filters_0, filters_1); + tableQuery.setFilter(filters_2); + ObTableQueryAndMutateRequest request_4 = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "update5" }); + ObPayload res_exec_4 = ((ObTableClient) client).execute(request_4); + res = (ObTableQueryAndMutateResult) res_exec_4; + Assert.assertEquals(1, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_4 = compareVal(ObCompareOp.GE, "c3", "update5"); + tableQuery.setFilter(confirm_4); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + // new test + // match the filter + ObTableValueFilter c3_EQ_rowX = compareVal(ObCompareOp.EQ, "c3", "z_row"); + MutationResult update_result = ((ObTableClient) client) + .update("test_query_filter_mutate").setRowKey(colVal("c1", 10L)) + .setFilter(c3_EQ_rowX) + .addMutateRow(row(colVal("c2", new byte[] { 1 }), colVal("c3", "update_ur"))) + .execute(); + Assert.assertEquals(1, update_result.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_ur = compareVal(ObCompareOp.EQ, "c3", "update_ur"); + tableQuery.setFilter(confirm_ur); + QueryResultSet result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // do not match the filter + update_result = ((ObTableClient) client).update("test_query_filter_mutate") + .setRowKey(colVal("c1", 10L)).setFilter(c3_EQ_rowX) + .addMutateRow(row(colVal("c2", new byte[] { 1 }), colVal("c3", "update_X"))) + .execute(); + Assert.assertEquals(0, update_result.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_x = compareVal(ObCompareOp.EQ, "c3", "update_X"); + tableQuery.setFilter(confirm_x); + result_ = tableQuery.execute(); + Assert.assertEquals(0, result_.cacheSize()); + + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + client.delete("test_query_filter_mutate", new Object[] { 3L }); + client.delete("test_query_filter_mutate", new Object[] { 4L }); + client.delete("test_query_filter_mutate", new Object[] { 5L }); + client.delete("test_query_filter_mutate", new Object[] { 10L }); + } + } + + @Test + public void testQueryFilter() throws Exception { + + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + + try { + + client.insert("test_query_filter_mutate", new Object[] { 0L }, new String[] { "c2", + "c3", "c4" }, new Object[] { new byte[] { 1 }, "row1", 10L }); + client.insert("test_query_filter_mutate", new Object[] { 1L }, new String[] { "c2", + "c3", "c4" }, new Object[] { new byte[] { 1 }, "row2", 11L }); + client.insert("test_query_filter_mutate", new Object[] { 2L }, new String[] { "c2", + "c3", "c4" }, new Object[] { new byte[] { 1 }, "row3", 12L }); + client.insert("test_query_filter_mutate", new Object[] { 3L }, new String[] { "c2", + "c3", "c4" }, new Object[] { new byte[] { 1 }, "row4", 13L }); + client.insert("test_query_filter_mutate", new Object[] { 4L }, new String[] { "c2", + "c3", "c4" }, new Object[] { new byte[] { 1 }, "row5", 14L }); + client.insert("test_query_filter_mutate", new Object[] { 5L }, new String[] { "c2", + "c3", "c4" }, new Object[] { new byte[] { 1 }, "row6", 15L }); + + TableQuery tableQuery = client.query("test_query_filter_mutate"); + /* Scan range must in one partition */ + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 200L }); + tableQuery.select("c1", "c2", "c3", "c4"); + + ObTableFilterList filterList; + // in/notin null cases + try { + ObTableInFilter inFilter = in("", 5); + } catch (Exception e) { + assertTrue(true); + } + + // in/notin null cases + try { + ObTableInFilter inFilter = in("xx", (Object) null); + } catch (Exception e) { + assertTrue(true); + } + + // in/notin null cases + try { + ObTableNotInFilter notInFilter = notIn("", 5); + } catch (Exception e) { + assertTrue(true); + } + + // in/notin null cases + try { + ObTableNotInFilter notInFilter = notIn("xx", (Object) null); + } catch (Exception e) { + assertTrue(true); + } + + // c1 in {0(short), 1(int), 2(long)} and c4 not in { 11 } + short num_16 = 0; + int num_32 = 1; + long num_64 = 2; + filterList = andList(in("c1", num_16, num_32, num_64), notIn("c4", 11)); + tableQuery.setFilter(filterList); + ObTableQueryAndMutateRequest request_0 = ((ObTableClient) client) + .obTableQueryAndUpdate(tableQuery, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "update1" }); + ObPayload res_exec_0 = ((ObTableClient) client).execute(request_0); + ObTableQueryAndMutateResult res = (ObTableQueryAndMutateResult) res_exec_0; + Assert.assertEquals(2, res.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + TableQuery confirmQuery = client.query("test_query_filter_mutate"); + confirmQuery.setFilter(compareVal(ObCompareOp.EQ, "c3", "update1")); + // 查询结果集 + QueryResultSet result = confirmQuery.select("c1", "c2", "c3") + .addScanRange(new Object[] { 0L }, new Object[] { 100L }).execute(); + long[] ans1 = { 0, 2 }; + for (int i = 0; i < 2; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get("c1"), ans1[i]); + System.out.println("c1:" + value.get("c1")); + } + Assert.assertFalse(result.next()); + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + client.delete("test_query_filter_mutate", new Object[] { 3L }); + client.delete("test_query_filter_mutate", new Object[] { 4L }); + client.delete("test_query_filter_mutate", new Object[] { 5L }); + client.delete("test_query_filter_mutate", new Object[] { 6L }); + } + } + + @Test + // Test ObTableValueFilter with IS/IS_NOT compareOp + public void testIsAndIsNot() throws Exception { + + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + String[] allColumnNames = new String[] { "c1", "c2", "c3", "c4" }; + String[] columnNames = new String[] { "c2", "c3", "c4" }; + String tableName = "test_query_filter_mutate"; + Object[] c1 = new Object[] { 7L, 8L, 9L }; + Object[] c2 = new Object[] { null, null, new byte[] { 3 } }; + Object[] c3 = new Object[] { "row7", "row8", "row9" }; + Object[] c4 = new Object[] { 10L, null, null }; + try { + for (int i = 0; i < c1.length; i++) { + client.insert(tableName, c1[i], columnNames, new Object[] { c2[i], c3[i], c4[i] }); + } + // IS compareOp with non-null value is not allowed + try { + ObTableFilter filter = compareVal(ObCompareOp.IS, "c1", "hello"); + fail(); + } catch (Exception e) { + assertTrue(true); + } + // IS_NOT compareOp with non-null value is not allowed + try { + ObTableFilter filter = compareVal(ObCompareOp.IS_NOT, "c1", "hello"); + fail(); + } catch (Exception e) { + assertTrue(true); + } + + // query with c2 is null + TableQuery tableQuery = client.query(tableName).select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }) + .setFilter(compareVal(ObCompareOp.IS, "c2", null)); + QueryResultSet result = tableQuery.execute(); + int expRowIdx[] = { 0, 1 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + + // query with c2 is not null + tableQuery = client.query(tableName).select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }) + .setFilter(compareVal(ObCompareOp.IS_NOT, "c2", null)); + result = tableQuery.execute(); + expRowIdx = new int[] { 2 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + + // query with c2 is null and c4 equals to 10 + tableQuery = client + .query(tableName) + .select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }) + .setFilter( + andList(compareVal(ObCompareOp.IS, "c2", null), + compareVal(ObCompareOp.EQ, "c4", 10L))); + result = tableQuery.execute(); + expRowIdx = new int[] { 0 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + + // query with c2 is null or c4 is not null + tableQuery = client + .query(tableName) + .select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }) + .setFilter( + orList(compareVal(ObCompareOp.IS, "c2", null), + compareVal(ObCompareOp.IS_NOT, "c4", null))); + result = tableQuery.execute(); + expRowIdx = new int[] { 0, 1 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + + // query with c2 is null and c4 is null + tableQuery = client + .query(tableName) + .select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }) + .setFilter( + andList(compareVal(ObCompareOp.IS, "c2", null), + compareVal(ObCompareOp.IS, "c4", null))); + result = tableQuery.execute(); + expRowIdx = new int[] { 1 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + } finally { + for (int i = 0; i < c1.length; i++) { + client.delete("test_query_filter_mutate", new Object[] { c1[i] }); + } + } + } + + @Test + public void testCompareWithNull() throws Exception { + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + + ObCompareOp ops[] = { ObCompareOp.LT, ObCompareOp.GT, ObCompareOp.LE, ObCompareOp.GE, + ObCompareOp.NE, ObCompareOp.EQ }; + // valueFilter with null value is not allowed except IS/IS_NOT comapreOp + for (ObCompareOp op : ops) { + try { + ObTableFilter filter = compareVal(op, "c1", null); + fail(); + } catch (Exception e) { + assertTrue(true); + } + } + + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + String[] allColumnNames = new String[] { "c1", "c2", "c3", "c4" }; + String[] columnNames = new String[] { "c2", "c3", "c4" }; + String tableName = "test_query_filter_mutate"; + Object[] c1 = new Object[] { 10L }; + Object[] c2 = new Object[] { null }; + Object[] c3 = new Object[] { null }; + Object[] c4 = new Object[] { null }; + Object vals[] = { new byte[] { 3 }, "row100", 10L }; + + try { + for (int i = 0; i < c1.length; i++) { + client.insert(tableName, c1[i], columnNames, new Object[] { c2[i], c3[i], c4[i] }); + } + + TableQuery tableQuery; + QueryResultSet result; + for (ObCompareOp op : ops) { + tableQuery = client + .query(tableName) + .select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }) + .setFilter( + orList(compareVal(op, "c2", vals[0]), compareVal(op, "c3", vals[1]), + compareVal(op, "c4", vals[2]))); + result = tableQuery.execute(); + Assert.assertEquals(result.cacheSize(), 0); + } + + } finally { + for (int i = 0; i < c1.length; i++) { + client.delete("test_query_filter_mutate", new Object[] { c1[i] }); + } + } + } + + @Test + // Test Query with filter and limit + public void testQueryFilterLimit() throws Exception { + + /* + * CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + * `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + * partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + * PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + ((ObTableClient) client) + .addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + String[] allColumnNames = new String[] { "c1", "c2", "c3", "c4" }; + String[] columnNames = new String[] { "c2", "c3", "c4" }; + String tableName = "test_query_filter_mutate"; + Object[] c1 = new Object[] { 11L, 12L, 13L, 14L, 15L }; + Object[] c2 = new Object[] { null, null, new byte[] { 3 }, new byte[] { 4 }, + new byte[] { 5 } }; + Object[] c3 = new Object[] { "row11", "row12", "row13", "row14", "row15" }; + Object[] c4 = new Object[] { 10L, null, null, null, null }; + try { + for (int i = 0; i < c1.length; i++) { + client.insert(tableName, c1[i], columnNames, new Object[] { c2[i], c3[i], c4[i] }); + } + + // only limit > 0 and offset >= 0 is allowed + try { + TableQuery tableQuery = client.query(tableName).select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }).limit(0, 0) + .setFilter(compareVal(ObCompareOp.IS, "c4", null)); + QueryResultSet result = tableQuery.execute(); + fail(); + } catch (Exception e) { + assertTrue(true); + } + try { + TableQuery tableQuery = client.query(tableName).select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }).limit(-1, 1) + .setFilter(compareVal(ObCompareOp.IS, "c4", null)); + QueryResultSet result = tableQuery.execute(); + fail(); + } catch (Exception e) { + assertTrue(true); + } + // IS_NOT compareOp with non-null value is not allowed + try { + ObTableFilter filter = compareVal(ObCompareOp.IS_NOT, "c1", "hello"); + fail(); + } catch (Exception e) { + assertTrue(true); + } + + // query with c4 is null ,limit is 2 and offset 1 + TableQuery tableQuery = client.query(tableName).select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }).limit(1, 2) + .setFilter(compareVal(ObCompareOp.IS, "c4", null)); + QueryResultSet result = tableQuery.execute(); + int expRowIdx[] = { 2, 3 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + + // query with c3 > "row12" ,limit 1 and offset 1 + tableQuery = client.query(tableName).select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }).limit(1, 1) + .setFilter(compareVal(ObCompareOp.GT, "c3", "row12")); + result = tableQuery.execute(); + expRowIdx = new int[] { 3 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + + // query with c3 != 'row13' and c4 is null, limit 1000 + tableQuery = client + .query(tableName) + .select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }) + .limit(1000) + .setFilter( + andList(compareVal(ObCompareOp.NE, "c3", "row13"), + compareVal(ObCompareOp.IS, "c4", null))); + + result = tableQuery.execute(); + expRowIdx = new int[] { 1, 3, 4 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + } finally { + for (int i = 0; i < c1.length; i++) { + client.delete("test_query_filter_mutate", new Object[] { c1[i] }); + } + } + } + + @Test + public void testMutation() throws Exception { + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + TableQuery tableQuery = client.query("test_mutation"); + tableQuery.addScanRange(new Object[] { 0L, "\0" }, new Object[] { 200L, "\254" }); + tableQuery.select("c1", "c2", "c3", "c4"); + + try { + // prepare data with insert + client.insert("test_mutation").setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 100L)).execute(); + client.insert("test_mutation").setRowKey(colVal("c1", 1L), colVal("c2", "row_1")) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 101L)).execute(); + client.insert("test_mutation").setRowKey(colVal("c1", 2L), colVal("c2", "row_2")) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 102L)).execute(); + client.insert("test_mutation").setRowKey(colVal("c1", 3L), colVal("c2", "row_3")) + .addMutateColVal(colVal("c1", 3L)).addMutateColVal(colVal("c2", "row_3")) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 103L)).execute(); + + // update / match filter + ObTableValueFilter c4_EQ_100 = compareVal(ObCompareOp.EQ, "c4", 100L); + MutationResult updateResult = client.update("test_mutation") + .setRowKey(colVal("c1", 0L), colVal("c2", "row_0")).setFilter(c4_EQ_100) + .addMutateRow(row(colVal("c3", new byte[] { 1 }), colVal("c4", 200L))).execute(); + Assert.assertEquals(1, updateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_1 = compareVal(ObCompareOp.EQ, "c4", 200L); + tableQuery.setFilter(confirm_1); + QueryResultSet result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // update / do not match filter + updateResult = client.update("test_mutation") + .setRowKey(colVal("c1", 1L), colVal("c2", "row_1")).setFilter(c4_EQ_100) + .addMutateRow(row(colVal("c3", new byte[] { 1 }), colVal("c4", 201L))).execute(); + Assert.assertEquals(0, updateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_2 = compareVal(ObCompareOp.EQ, "c4", 201L); + tableQuery.setFilter(confirm_2); + result_ = tableQuery.execute(); + Assert.assertEquals(0, result_.cacheSize()); + + // update / duplicate rowkey between setrowkey and mutatecolumns + updateResult = client.update("test_mutation") + .setRowKey(colVal("c1", 1L), colVal("c2", "row_1")).setFilter(c4_EQ_100) + .addMutateColVal(colVal("c1", 1L), colVal("c2", "row_1")) + .addMutateRow(row(colVal("c3", new byte[] { 1 }), colVal("c4", 201L))).execute(); + Assert.assertEquals(0, updateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + confirm_2 = compareVal(ObCompareOp.EQ, "c4", 201L); + tableQuery.setFilter(confirm_2); + result_ = tableQuery.execute(); + Assert.assertEquals(0, result_.cacheSize()); + + // delete / do not match filter + ObTableValueFilter c4_EQ_103 = compareVal(ObCompareOp.EQ, "c4", 103L); + MutationResult deleteResult = client.delete("test_mutation") + .setRowKey(colVal("c1", 2L), colVal("c2", "row_2")).setFilter(c4_EQ_103).execute(); + Assert.assertEquals(0, deleteResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + tableQuery.setFilter(c4_EQ_103); + result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // replace + MutationResult replaceResult = client.replace("test_mutation") + .setRowKey(colVal("c1", 2L), colVal("c2", "row_2")) + .addMutateRow(row(colVal("c3", new byte[] { 2 }), colVal("c4", 202L))).execute(); + Assert.assertEquals(2, replaceResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_3 = compareVal(ObCompareOp.EQ, "c4", 202L); + tableQuery.setFilter(confirm_3); + result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // InsertOrUpdate / Insert + MutationResult insertOrUpdateResult = client.insertOrUpdate("test_mutation") + .setRowKey(colVal("c1", 4L), colVal("c2", "row_4")) + .addMutateRow(row(colVal("c3", new byte[] { 2 }), colVal("c4", 104L))).execute(); + Assert.assertEquals(1, insertOrUpdateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_4 = compareVal(ObCompareOp.EQ, "c4", 104L); + tableQuery.setFilter(confirm_4); + result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // InsertOrUpdate / Update + insertOrUpdateResult = client.insertOrUpdate("test_mutation") + .setRowKey(colVal("c1", 4L), colVal("c2", "row_4")) + .addMutateRow(row(colVal("c3", new byte[1]), colVal("c4", 104L))).execute(); + Assert.assertEquals(1, insertOrUpdateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + confirm_4 = compareVal(ObCompareOp.EQ, "c4", 104L); + tableQuery.setFilter(confirm_4); + result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // InsertOrUpdate / Update / duplicate rowkey between setrowkey and mutatecolumns + insertOrUpdateResult = client.insertOrUpdate("test_mutation") + .setRowKey(colVal("c1", 4L), colVal("c2", "row_4")) + .addMutateColVal(colVal("c1", 4L), colVal("c2", "row_4")) + .addMutateRow(row(colVal("c3", new byte[1]), colVal("c4", 104L))).execute(); + Assert.assertEquals(1, insertOrUpdateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + confirm_4 = compareVal(ObCompareOp.EQ, "c4", 104L); + tableQuery.setFilter(confirm_4); + result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // increment / without filter + // result will send back the latest mutated column + MutationResult incrementResult = client.increment("test_mutation") + .setRowKey(colVal("c1", 3L), colVal("c2", "row_3")) + .addMutateColVal(colVal("c4", 100L)).execute(); + Assert.assertEquals(1, incrementResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_5 = compareVal(ObCompareOp.EQ, "c4", 203L); + tableQuery.setFilter(confirm_5); + result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // increment / with filter hit + // result will send back the value before increment + incrementResult = client.increment("test_mutation") + .setRowKey(colVal("c1", 3L), colVal("c2", "row_3")) + .addMutateColVal(colVal("c4", 100L)) + .setFilter(andList(compareVal(ObCompareOp.EQ, "c4", 203L))).execute(); + Assert.assertEquals(1, incrementResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_6 = compareVal(ObCompareOp.EQ, "c4", 303L); + tableQuery.setFilter(confirm_6); + result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // increment / with filter not hit + incrementResult = client.increment("test_mutation") + .setRowKey(colVal("c1", 3L), colVal("c2", "row_3")) + .addMutateColVal(colVal("c4", 100L)) + .setFilter(andList(compareVal(ObCompareOp.EQ, "c4", 203L))).execute(); + Assert.assertEquals(0, incrementResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_7 = compareVal(ObCompareOp.EQ, "c4", 303L); + tableQuery.setFilter(confirm_7); + result_ = tableQuery.execute(); + Assert.assertEquals(1, result_.cacheSize()); + + // append / without filter + // result will send back the latest mutated column + MutationResult appendResult = client.append("test_mutation") + .setRowKey(colVal("c1", 4L), colVal("c2", "row_4")) + .addMutateColVal(colVal("c3", new byte[1])).execute(); + Assert.assertEquals(1, appendResult.getAffectedRows()); + + // append / with filter hit + // result will send back the value before append + appendResult = client.append("test_mutation") + .setRowKey(colVal("c1", 4L), colVal("c2", "row_4")) + .addMutateColVal(colVal("c3", new byte[1])) + .setFilter(andList(compareVal(ObCompareOp.EQ, "c4", 104L))).execute(); + Assert.assertEquals(1, appendResult.getAffectedRows()); + + // append / with filter not hit + appendResult = client.append("test_mutation") + .setRowKey(colVal("c1", 3L), colVal("c2", "row_3")) + .addMutateColVal(colVal("c3", new byte[0])) + .setFilter(andList(compareVal(ObCompareOp.EQ, "c4", 203L))).execute(); + Assert.assertEquals(0, appendResult.getAffectedRows()); + + // increment non-integer column without filter + try { + incrementResult = client.increment("test_mutation") + .setRowKey(colVal("c1", 0L), colVal("c2", "row_0")) + .addMutateColVal(colVal("c3", 100L)).execute(); + Assert.assertTrue(false); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, + ((ObTableException) e).getErrorCode()); + } + // increment non-integer column with filter + try { + incrementResult = client.increment("test_mutation") + .setRowKey(colVal("c1", 0L), colVal("c2", "row_0")) + .addMutateColVal(colVal("c3", 100L)) + .setFilter(compareVal(ObCompareOp.EQ, "c4", 200L)).execute(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, + ((ObTableException) e).getErrorCode()); + } + + // increment integer column with string + try { + incrementResult = client.increment("test_mutation") + .setRowKey(colVal("c1", 0L), colVal("c2", "row_0")) + .addMutateColVal(colVal("c4", "hello world")) + .setFilter(compareVal(ObCompareOp.EQ, "c4", 200L)).execute(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_COLUMN_TYPE_NOT_MATCH.errorCode, + ((ObTableException) e).getErrorCode()); + } + + // append non-string column without filter + try { + appendResult = client.append("test_mutation") + .setRowKey(colVal("c1", 0L), colVal("c2", "row_0")) + .addMutateColVal(colVal("c4", new byte[1])).execute(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, + ((ObTableException) e).getErrorCode()); + } + + // append non-string column with filter + try { + appendResult = client.append("test_mutation") + .setRowKey(colVal("c1", 0L), colVal("c2", "row_0")) + .addMutateColVal(colVal("c4", new byte[1])) + .setFilter(compareVal(ObCompareOp.EQ, "c4", 200L)).execute(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, + ((ObTableException) e).getErrorCode()); + } + + // append string column with integer + try { + appendResult = client.append("test_mutation") + .setRowKey(colVal("c1", 0L), colVal("c2", "row_0")) + .addMutateColVal(colVal("c3", new byte[1])) + .setFilter(compareVal(ObCompareOp.EQ, "c4", 200L)).execute(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_OBJ_TYPE_ERROR.errorCode, + ((ObTableException) e).getErrorCode()); + } + } finally { + client.delete("test_mutation").setRowKey(colVal("c1", 0L), colVal("c2", "row_0")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 1L), colVal("c2", "row_1")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 2L), colVal("c2", "row_2")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 3L), colVal("c2", "row_3")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 4L), colVal("c2", "row_4")) + .execute(); + } + } + + @Test + public void testPut() throws Exception { + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + try { + // put + MutationResult insertOrUpdateResult = client.put("test_mutation") + .setRowKey(colVal("c1", 1L), colVal("c2", "row_1")) + .addMutateRow(row(colVal("c3", new byte[] { 2 }), colVal("c4", 1L))).execute(); + Assert.assertEquals(1, insertOrUpdateResult.getAffectedRows()); + + // check + Map res = client.get("test_mutation", new Object[] { 1L, "row_1" }, + null); + Assert.assertEquals(1L, res.get("c1")); + Assert.assertEquals("row_1", res.get("c2")); + Assert.assertEquals(1L, res.get("c4")); + + // use put but not set all column, cause exception + try { + insertOrUpdateResult = client.put("test_mutation") + .setRowKey(colVal("c1", 1L), colVal("c2", "row_1")) + .addMutateRow(row(colVal("c3", new byte[] { 2 }))).execute(); + } catch (Exception e) { + Assert.assertTrue(e instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, + ((ObTableException) e).getErrorCode()); + } + } finally { + client.delete("test_mutation").setRowKey(colVal("c1", 1L), colVal("c2", "row_1")) + .execute(); + } + } + + @Test + public void testBatchMutation() throws Exception { + if (ObGlobal.isLsOpSupport()) { + return; + } + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + TableQuery tableQuery = client.query("test_mutation"); + tableQuery.addScanRange(new Object[] { 0L, "\0" }, new Object[] { 200L, "\254" }); + tableQuery.select("c1", "c2", "c3", "c4"); + + try { + // prepare data with insert + client.insert("test_mutation").setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 100L)).execute(); + client.insert("test_mutation").setRowKey(colVal("c1", 1L), colVal("c2", "row_1")) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 101L)).execute(); + client.insert("test_mutation").setRowKey(colVal("c1", 2L), colVal("c2", "row_2")) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 102L)).execute(); + client.insert("test_mutation").setRowKey(colVal("c1", 3L), colVal("c2", "row_3")) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 103L)).execute(); + + Insert insert_0 = insert().setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 100L)); + Insert insert_1 = insert().setRowKey(row(colVal("c1", 4L), colVal("c2", "row_4"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 104L)); + Update update_0 = update().setRowKey(row(colVal("c1", 4L), colVal("c2", "row_4"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 204L)); + TableQuery query_0 = query().setRowKey(row(colVal("c1", 4L), colVal("c2", "row_4"))) + .select("c3", "c4"); + + BatchOperationResult batchResult = client.batchOperation("test_mutation") + .addOperation(insert_0, insert_1, update_0).addOperation(query_0) + .setIsAtomic(false).execute(); + Assert.assertEquals(1, batchResult.getWrongCount()); + Assert.assertEquals(3, batchResult.getCorrectCount()); + Assert.assertEquals(0, batchResult.getWrongIdx()[0]); + Assert.assertEquals(1, batchResult.getCorrectIdx()[0]); + Assert.assertEquals(1, batchResult.get(1).getAffectedRows()); + Assert.assertEquals(1, batchResult.get(2).getAffectedRows()); + OperationResult opResult = batchResult.get(3); + Assert.assertEquals(204L, opResult.getOperationRow().get("c4")); + Assert.assertEquals(204L, batchResult.get(3).getOperationRow().get("c4")); + opResult = batchResult.get(2); + Assert.assertNull(opResult.getOperationRow().get("c4")); + Assert.assertNull(batchResult.get(2).getOperationRow().get("c4")); + + long[] c1Vals = { 0L, 1L, 2L }; + String[] c2Vals = { "row_0", "row_1", "row_2" }; + byte[] c3Val = new byte[] { 1 }; + long[] c4Vals = { 100L, 101L, 102L }; + BatchOperation batchOperation = client.batchOperation("test_mutation"); + for (int i = 0; i < c1Vals.length; i++) { + Row rowKey1 = row(colVal("c1", c1Vals[i]), colVal("c2", c2Vals[i])); + TableQuery query = query().setRowKey(rowKey1).select("c1", "c2", "c3", "c4"); + batchOperation.addOperation(query); + } + BatchOperationResult result = batchOperation.execute(); + for (int i = 0; i < c2Vals.length; i++) { + Row row = result.get(i).getOperationRow(); + Assert.assertEquals(4, row.size()); + Assert.assertEquals(c1Vals[i], row.get("c1")); + Assert.assertEquals(c2Vals[i], row.get("c2")); + Assert.assertTrue(Arrays.equals(c3Val, (byte[]) row.get("c3"))); + Assert.assertEquals(c4Vals[i], row.get("c4")); + } + + // test duplicate rowkey in mutatecolval and rowkey + // mutation will automatically remove duplicate key in rowkey and mutatecolval + Insert insert_2 = insert().setRowKey(row(colVal("c1", 5L), colVal("c2", "row_5"))) + .addMutateColVal(colVal("c1", 5L), colVal("c2", "row_5")) + .addMutateColVal(colVal("c3", new byte[] { 1 })).addMutateColVal(colVal("c4", 5L)); + Update update_1 = update().setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateRow(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })).addMutateColVal(colVal("c4", 0L)); + InsertOrUpdate iou_0 = insertOrUpdate() + .setRowKey(row(colVal("c1", 1L), colVal("c2", "row_1"))) + .addMutateColVal(colVal("c2", "row_1")) + .addMutateColVal(colVal("c3", new byte[] { 1 })).addMutateColVal(colVal("c4", 0L)); + + batchResult = client.batchOperation("test_mutation") + .addOperation(insert_2, update_1, iou_0).execute(); + Assert.assertEquals(0, batchResult.getWrongCount()); + Assert.assertEquals(3, batchResult.getCorrectCount()); + Assert.assertEquals(0, batchResult.getCorrectIdx()[0]); + Assert.assertEquals(1, batchResult.get(1).getAffectedRows()); + Assert.assertEquals(1, batchResult.get(2).getAffectedRows()); + + // test duplicate rowkey update + Update update_2 = update().setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 10 })).addMutateColVal(colVal("c4", 0L)); + InsertOrUpdate iou_1 = insertOrUpdate() + .setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })).addMutateColVal(colVal("c4", 1L)); + InsertOrUpdate iou_2 = insertOrUpdate() + .setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 2 })).addMutateColVal(colVal("c4", 2L)); + Increment inc_0 = increment().setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateRow(row(colVal("c4", 100L))); + Append apd_0 = append().setRowKey(row(colVal("c1", 0L), colVal("c2", "row_0"))) + .addMutateRow(row(colVal("c3", new byte[] { 0 }))); + + batchResult = client.batchOperation("test_mutation") + .addOperation(update_2, iou_1, iou_2, inc_0, apd_0).execute(); + Assert.assertEquals(0, batchResult.getWrongCount()); + Assert.assertEquals(5, batchResult.getCorrectCount()); + Assert.assertEquals(0, batchResult.getCorrectIdx()[0]); + Assert.assertEquals(1, batchResult.get(1).getAffectedRows()); + Assert.assertEquals(1, batchResult.get(2).getAffectedRows()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + client.delete("test_mutation").setRowKey(colVal("c1", 0L), colVal("c2", "row_0")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 1L), colVal("c2", "row_1")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 2L), colVal("c2", "row_2")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 3L), colVal("c2", "row_3")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 4L), colVal("c2", "row_4")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 5L), colVal("c2", "row_5")) + .execute(); + } + } + + @Test + public void testMutationWithScanRange() throws Exception { + + // CREATE TABLE `test_mutation_with_range` ( + // `c1` bigint NOT NULL, + // `c1sk` varchar(20) DEFAULT NULL, + // `c2` varbinary(1024) DEFAULT NULL, + // `c3` varchar(20) DEFAULT NULL, + // `c4` bigint DEFAULT NULL, + // PRIMARY KEY(`c1`, `c1sk`)) partition by range columns (`c1`) ( + // PARTITION p0 VALUES LESS THAN (300), + // PARTITION p1 VALUES LESS THAN (1000), + // PARTITION p2 VALUES LESS THAN MAXVALUE); + + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + try { + ((ObTableClient) client).addRowKeyElement("test_mutation_with_range", new String[] { + "c1", "c1sk" }); + client.insert("test_mutation_with_range", new Object[] { 0L, "c0" }, new String[] { + "c2", "c3" }, new Object[] { new byte[] { 1 }, "row1" }); + client.insert("test_mutation_with_range", new Object[] { 1L, "c1" }, new String[] { + "c2", "c3" }, new Object[] { new byte[] { 1 }, "row2" }); + client.insert("test_mutation_with_range", new Object[] { 2L, "c2" }, new String[] { + "c2", "c3" }, new Object[] { new byte[] { 1 }, "row3" }); + client.insert("test_mutation_with_range", new Object[] { 3L, "c3" }, new String[] { + "c2", "c3", "c4" }, new Object[] { new byte[] { 1 }, "row4", 4L }); + client.insert("test_mutation_with_range", new Object[] { 4L, "c4" }, new String[] { + "c2", "c3", "c4" }, new Object[] { new byte[] { 1 }, "row5", 4L }); + client.insert("test_mutation_with_range", new Object[] { 5L, "c5" }, new String[] { + "c2", "c3" }, new Object[] { new byte[] { 1 }, "row6" }); + client.insert("test_mutation_with_range") + .setRowKey(row(colVal("c1", 10L), colVal("c1sk", "c10"))) + .addMutateColVal(new ColumnValue("c2", new byte[] { 1 })) + .addMutateColVal(new ColumnValue("c3", "z_row")) + .addMutateColVal(new ColumnValue("c4", 10L)).execute(); + + TableQuery tableQuery = client.query("test_mutation_with_range"); + /* Scan range must in one partition */ + tableQuery.addScanRange(new Object[] { 0L, "A" }, new Object[] { 200L, "z" }); + tableQuery.select("c1", "c2", "c3", "c4"); + + ObTableValueFilter c1_EQ_0 = compareVal(ObCompareOp.EQ, "c1", 0); + ObTableValueFilter c1_LE_4 = compareVal(ObCompareOp.LE, "c1", 4); + ObTableValueFilter c1_GE_3 = compareVal(ObCompareOp.GE, "c1", 3); + ObTableFilterList filters_0 = andList(); + ObTableFilterList filters_1 = andList(); + + // c1 = 0 && c1 = 0 + filters_0.addFilter(c1_EQ_0, c1_EQ_0); + MutationResult updateResult = client.update("test_mutation_with_range") + .setFilter(filters_0) + .addMutateRow(row(colVal("c2", new byte[] { 1 }), colVal("c3", "update1"))) + .setScanRangeColumns("c1", "c1sk") + .addScanRange(new Object[] { 0L, "A" }, new Object[] { 200L, "z" }).execute(); + Assert.assertEquals(1, updateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_0 = compareVal(ObCompareOp.EQ, "c3", "update1"); + tableQuery.setFilter(confirm_0); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + + // c1 >= 3 && c1 <= 4 ( scan range c1 > 4 ) + filters_1.addFilter(c1_GE_3, c1_LE_4); + updateResult = client.increment("test_mutation_with_range").setFilter(filters_1) + .addMutateRow(row(colVal("c4", 100L))).setScanRangeColumns("c1", "c1sk") + .addScanRange(new Object[] { 4L, "A" }, new Object[] { 200L, "z" }).execute(); + Assert.assertEquals(1, updateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_1 = compareVal(ObCompareOp.EQ, "c4", 104L); + tableQuery.setFilter(confirm_1); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + + // only scan + if (((ObTableClient) client).isOdpMode()) { + updateResult = client.update("test_mutation_with_range") + .addMutateRow(row(colVal("c2", new byte[] { 1 }), colVal("c3", "update2"))) + .setScanRangeColumns("c1", "c1sk") + .addScanRange(new Object[] { 4L, "A" }, new Object[] { 9L, "z" }).execute(); + Assert.assertEquals(2, updateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_2 = compareVal(ObCompareOp.EQ, "c3", "update2"); + tableQuery.setFilter(confirm_2); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + try { + updateResult = client.update("test_mutation_with_range") + .addMutateRow(row(colVal("c2", new byte[] { 1 }), colVal("c3", "update2"))) + .setFilter(filters_0).execute(); + Assert.assertTrue(false); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(e instanceof ObTableException); + if (client instanceof ObTableClient && ((ObTableClient) client).isOdpMode()) { + Assert.assertEquals(ResultCodes.OB_ERR_UNEXPECTED.errorCode, + ((ObTableUnexpectedException) e).getErrorCode()); + } else { + Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, + ((ObTableException) e).getErrorCode()); + } + } + } else { + updateResult = client.update("test_mutation_with_range") + .addMutateRow(row(colVal("c2", new byte[] { 1 }), colVal("c3", "update2"))) + .setScanRangeColumns("c1", "c1sk") + .addScanRange(new Object[] { 4L, "A" }, new Object[] { 9L, "z" }).execute(); + Assert.assertEquals(2, updateResult.getAffectedRows()); + /* To confirm changing. re-query to get the latest data */ + ObTableValueFilter confirm_2 = compareVal(ObCompareOp.EQ, "c3", "update2"); + tableQuery.setFilter(confirm_2); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + } + } finally { + client.delete("test_mutation_with_range", new Object[] { 0L, "c0" }); + client.delete("test_mutation_with_range", new Object[] { 1L, "c1" }); + client.delete("test_mutation_with_range", new Object[] { 2L, "c2" }); + client.delete("test_mutation_with_range", new Object[] { 3L, "c3" }); + client.delete("test_mutation_with_range", new Object[] { 4L, "c4" }); + client.delete("test_mutation_with_range", new Object[] { 5L, "c5" }); + client.delete("test_mutation_with_range", new Object[] { 10L, "c10" }); + } + } + + @Test + public void testMultiThreadBatchOperation() throws Exception { + try { + int threadNum = 16; + List allWorkers = new ArrayList<>(); + long startTime = System.currentTimeMillis(); + for (int i = 0; i < threadNum; ++i) { + ObTableHotkeyThrottleUtil worker = new ObTableHotkeyThrottleUtil(); + worker.init(threadNum, i, startTime, "test_throttle", new String[] { "c1", "c2" }, ObTableHotkeyThrottleUtil.TestType.random, ObTableHotkeyThrottleUtil.OperationType.batchOperation, 100, this.client, 16); + allWorkers.add(worker); + worker.start(); + } + for (int i = 0; i < threadNum; ++i) { + allWorkers.get(i).join(); + System.out.println("Thread " + i + "th has finished"); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testAtomicBatchMutation() throws Exception { + try { + // atomic batch operation can not span partitions + BatchOperation batchOperation = client.batchOperation("test_mutation"); + Insert insert_0 = insert().setRowKey(row(colVal("c1", 100L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 100L)); + Insert insert_1 = insert().setRowKey(row(colVal("c1", 400L), colVal("c2", "row_1"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 100L)); + BatchOperationResult result = batchOperation.addOperation(insert_0, insert_1) + .setIsAtomic(true).execute(); + } catch (Exception e) { + e.printStackTrace(); + if (client instanceof ObTableClient && ((ObTableClient) client).isOdpMode()) { + Assert.assertTrue(e instanceof ObTableUnexpectedException); + } else { + Assert.assertTrue(e instanceof ObTablePartitionConsistentException); + } + } finally { + client.delete("test_mutation").setRowKey(colVal("c1", 100L), colVal("c2", "row_0")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 400L), colVal("c2", "row_1")) + .execute(); + } + + try { + BatchOperation batchOperation = client.batchOperation("test_mutation"); + Insert insert_0 = insert().setRowKey(row(colVal("c1", 100L), colVal("c2", "row_0"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 100L)); + Insert insert_1 = insert().setRowKey(row(colVal("c1", 200L), colVal("c2", "row_1"))) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", 100L)); + BatchOperationResult result = batchOperation.addOperation(insert_0, insert_1) + .setIsAtomic(true).execute(); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + client.delete("test_mutation").setRowKey(colVal("c1", 100L), colVal("c2", "row_0")) + .execute(); + client.delete("test_mutation").setRowKey(colVal("c1", 200L), colVal("c2", "row_1")) + .execute(); + } + } + + @Test + public void testDateTime() throws Exception { + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_datetime_table", new String[] { "c1" }); + /* + CREATE TABLE IF NOT EXISTS `test_datetime_table` ( + `c1` varchar(20) NOT NULL, + `c2` datetime(6) DEFAULT NULL, + `c3` datetime(3) DEFAULT NULL, + `c4` datetime DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + */ + + SimpleDateFormat sdf = new SimpleDateFormat(" yyyy-MM-dd HH:mm:ss "); + // 1650513600001 -> 2022-04-21 12:00:00.001 + Date date1 = new Date(1650513600001L); + // 1650524400001 -> 2022-04-21 15:00:00.001 for c2/c3 + Date date2 = new Date(1650524400001L); + // 1650524400000 -> 2022-04-21 15:00:00 for c4 + Date date3 = new Date(1650524400000L); + try { + client.delete("test_datetime_table").setRowKey(colVal("c1", "0")).execute(); + client.delete("test_datetime_table").setRowKey(colVal("c1", "1")).execute(); + + // client.insert("test_datetime_table", "0", new String[] { "c2" }, new Object[] { date1 }); + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement + .execute("insert into test_datetime_table values (0, '2022-04-21 12:00:00.001', '2022-04-21 12:00:00.001', '2022-04-21 12:00:00.001')"); + + // 2022-04-21 12:00:00.001 CST = 1650513600001 -> insert by sql, get by obkv client + Map res = client.get("test_datetime_table", "0", new String[] { "c2", + "c3", "c4" }); + Assert.assertEquals(date1, res.get("c2")); + Assert.assertEquals(date1, res.get("c3")); + Assert.assertNotEquals(date1, res.get("c4")); // datetime -> second + + // 2022-04-21 15:00:00.001 CST = 1650524400001 -> insert by obkv client, get by sql + client.insert("test_datetime_table", "1", new String[] { "c2", "c3", "c4" }, + new Object[] { date2, date2, date3 }); + ResultSet resultSet = statement + .executeQuery("select * from test_datetime_table where c1 = 1"); + resultSet.next(); + // since getDate will miss micro second in mysql, we use getTimestamp here + Assert.assertEquals(date2, resultSet.getTimestamp("c2")); + Assert.assertEquals(date2, resultSet.getTimestamp("c3")); + Assert.assertEquals(date3, resultSet.getTimestamp("c4")); + } finally { + client.delete("test_datetime_table").setRowKey(colVal("c1", "0")).execute(); + client.delete("test_datetime_table").setRowKey(colVal("c1", "1")).execute(); + } + } + + /** + table : + create table cse_index_1 ( + measurement VARBINARY(1024) NOT NULL, + tag_key VARBINARY(1024) NOT NULL, + tag_value VARBINARY(1024) NOT NULL, + series_ids MEDIUMBLOB NOT NULL, + PRIMARY KEY(measurement, tag_key, tag_value)) + partition by key(measurement) partitions 13; + **/ + + @Test + public void testBatchInsertJudge() throws Exception { + + try { + cleanTable("cse_index_1"); + // prepare data with insert + Insert insert_0 = insert().setRowKey( + row(colVal("measurement", "measurement1"), colVal("tag_key", "tag_key1"), + colVal("tag_value", "tag_value1"))).addMutateColVal( + colVal("series_ids", "series_ids1")); + Insert insert_1 = insert().setRowKey( + row(colVal("measurement", "measurement1"), colVal("tag_key", "tag_key2"), + colVal("tag_value", "tag_value2"))).addMutateColVal( + colVal("series_ids", "series_ids2")); + Insert insert_2 = insert().setRowKey( + row(colVal("measurement", "measurement1"), colVal("tag_key", "tag_key3"), + colVal("tag_value", "tag_value3"))).addMutateColVal( + colVal("series_ids", "series_ids3")); + BatchOperationResult batchResult = client.batchOperation("cse_index_1") + .addOperation(insert_0).addOperation(insert_1).addOperation(insert_2).execute(); + Assert.assertEquals(0, batchResult.getWrongCount()); + Assert.assertEquals(3, batchResult.getCorrectCount()); + + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanTable("cse_index_1"); + } + } + + @Test + // Test Query with filter and limit + public void testQueryOffsetWithoutLimit() throws Exception { + + /* + CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + String[] allColumnNames = new String[] { "c1", "c2", "c3", "c4" }; + String[] columnNames = new String[] { "c2", "c3", "c4" }; + String tableName = "test_query_filter_mutate"; + Object[] c1 = new Object[] { 11L, 12L, 13L, 14L, 15L }; + Object[] c2 = new Object[] { null, null, new byte[] { 3 }, new byte[] { 4 }, + new byte[] { 5 } }; + Object[] c3 = new Object[] { "row11", "row12", "row13", "row14", "row15" }; + Object[] c4 = new Object[] { 10L, null, null, null, null }; + try { + for (int i = 0; i < c1.length; i++) { + client.insert(tableName, c1[i], columnNames, new Object[] { c2[i], c3[i], c4[i] }); + } + + // query with c4 is null ,limit is 2 and offset 1 + TableQuery tableQuery = client.query(tableName).select(allColumnNames) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }).limit(2, -1); + QueryResultSet result = tableQuery.execute(); + int expRowIdx[] = { 2, 3 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertEquals("offset can not be use without limit", + ((ObTableException) e).getMessage()); + } finally { + for (int i = 0; i < c1.length; i++) { + client.delete("test_query_filter_mutate", new Object[] { c1[i] }); + } + } + } + + @Test + // Test Query with filter and limit + public void testAsyncQueryOffsetWithoutLimit() throws Exception { + + /* + CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + final ObTableClient client = (ObTableClient) this.client; + client.addRowKeyElement("test_query_filter_mutate", new String[] { "c1" }); //同索引列的值一样 + String[] allColumnNames = new String[] { "c1", "c2", "c3", "c4" }; + String[] columnNames = new String[] { "c2", "c3", "c4" }; + String tableName = "test_query_filter_mutate"; + Object[] c1 = new Object[] { 11L, 12L, 13L, 14L, 15L }; + Object[] c2 = new Object[] { null, null, new byte[] { 3 }, new byte[] { 4 }, + new byte[] { 5 } }; + Object[] c3 = new Object[] { "row11", "row12", "row13", "row14", "row15" }; + Object[] c4 = new Object[] { 10L, null, null, null, null }; + try { + for (int i = 0; i < c1.length; i++) { + client.insert(tableName, c1[i], columnNames, new Object[] { c2[i], c3[i], c4[i] }); + } + + // query with c4 is null ,limit is 2 and offset 1 + TableQuery tableQuery = client.query(tableName).select(allColumnNames).setBatchSize(1) + .addScanRange(new Object[] { 0L }, new Object[] { 200L }).limit(2, -1); + QueryResultSet result = tableQuery.asyncExecute(); + int expRowIdx[] = { 2, 3 }; + Assert.assertEquals(result.cacheSize(), expRowIdx.length); + for (int i = 0; i < expRowIdx.length; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get(allColumnNames[0]), c1[expRowIdx[i]]); + assertTrue(Arrays.equals((byte[]) value.get(allColumnNames[1]), + (byte[]) c2[expRowIdx[i]])); + assertEquals(value.get(allColumnNames[2]), c3[expRowIdx[i]]); + assertEquals(value.get(allColumnNames[3]), c4[expRowIdx[i]]); + } + Assert.assertFalse(result.next()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertEquals("offset can not be use without limit", + ((ObTableException) e).getMessage()); + } finally { + for (int i = 0; i < c1.length; i++) { + client.delete("test_query_filter_mutate", new Object[] { c1[i] }); + } + } + } + + @Test + public void testQueryWithFilterColumn() throws Exception { + // test filter column not in select columns, + /* + CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + final String TABLE_NAME = "test_query_filter_mutate"; + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client).addRowKeyElement(TABLE_NAME, new String[] { "c1" }); //同索引列的值一样 + + try { + client.insert(TABLE_NAME, new Object[] { 0L }, new String[] { "c2", "c3" }, + new Object[] { new byte[] { 1 }, "row1" }); + client.insert(TABLE_NAME, new Object[] { 1L }, new String[] { "c2", "c3" }, + new Object[] { new byte[] { 1 }, "row2" }); + client.insert(TABLE_NAME, new Object[] { 2L }, new String[] { "c2", "c3" }, + new Object[] { new byte[] { 1 }, "row3" }); + + TableQuery tableQuery = client.query(TABLE_NAME); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 250L }); + tableQuery.select("c1", "c2"); + + ObTableValueFilter filter_0 = new ObTableValueFilter(ObCompareOp.GT, "c3", "row1"); + + tableQuery.setFilter(filter_0); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + } + } + + @Test + public void testAsyncQueryWithFilterColumn() throws Exception { + // test filter column not in select columns, + /* + CREATE TABLE `test_query_filter_mutate` (`c1` bigint NOT NULL, `c2` varbinary(1024) DEFAULT NULL, + `c3` varchar(20) DEFAULT NULL,`c4` bigint DEFAULT NULL, PRIMARY KEY(`c1`)) + partition by range columns (`c1`) ( PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + final String TABLE_NAME = "test_query_filter_mutate"; + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + ((ObTableClient) client).addRowKeyElement(TABLE_NAME, new String[] { "c1" }); //同索引列的值一样 + + try { + client.insert(TABLE_NAME, new Object[] { 0L }, new String[] { "c2", "c3" }, + new Object[] { new byte[] { 1 }, "row1" }); + client.insert(TABLE_NAME, new Object[] { 1L }, new String[] { "c2", "c3" }, + new Object[] { new byte[] { 1 }, "row2" }); + client.insert(TABLE_NAME, new Object[] { 2L }, new String[] { "c2", "c3" }, + new Object[] { new byte[] { 1 }, "row3" }); + + TableQuery tableQuery = client.query(TABLE_NAME); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 250L }); + tableQuery.select("c1", "c2"); + tableQuery.setBatchSize(1); + + ObTableValueFilter filter_0 = new ObTableValueFilter(ObCompareOp.GT, "c3", "row1"); + + tableQuery.setFilter(filter_0); + QueryResultSet result = tableQuery.asyncExecute(); + int expected_ret = 2; + while (result.next()) { + expected_ret -= 1; + } + Assert.assertEquals(0, expected_ret); + } finally { + client.delete("test_query_filter_mutate", new Object[] { 0L }); + client.delete("test_query_filter_mutate", new Object[] { 1L }); + client.delete("test_query_filter_mutate", new Object[] { 2L }); + } + } + + @Test + public void testTimeStamp() throws Exception { + /* + CREATE TABLE IF NOT EXISTS `test_timestamp_table` ( + `c1` varchar(20) NOT NULL, + `c2` timestamp(6) DEFAULT NULL, + `c3` timestamp(3) DEFAULT NULL, + `c4` timestamp DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + */ + final String TABLE_NAME = "test_timestamp_table"; + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + ((ObTableClient) client).addRowKeyElement(TABLE_NAME, new String[] { "c1" }); //同索引列的值一样 + try { + LocalDateTime localDateTime = LocalDateTime.of(1997, 7, 1, 0, 0); + ZonedDateTime zonedDateTime = localDateTime.atOffset(ZoneOffset.UTC).toZonedDateTime(); + long seconds = zonedDateTime.toEpochSecond(); + int nanos = zonedDateTime.getNano() + 1000; // 1 micro second + long milliseconds = seconds * 1_000; + Timestamp timestamp6 = new Timestamp(milliseconds); + Timestamp timestamp3 = new Timestamp(milliseconds); + Timestamp timestamp = new Timestamp(milliseconds); + // Timestamp: 1997-07-01 08:00:00.000001 + timestamp6.setNanos(nanos); + + client.insert(TABLE_NAME, new Object[] { "key_0" }, new String[] { "c2", "c3", "c4" }, + new Object[] { timestamp6, timestamp3, timestamp }); + + Map values = client.get("test_timestamp_table", "key_0", + new String[] { "c2", "c3", "c4" }); + + assertEquals(timestamp6, values.get("c2")); + assertEquals(timestamp3, values.get("c3")); + assertEquals(timestamp, values.get("c4")); + assertEquals(timestamp6.toString(), values.get("c2").toString()); + } finally { + client.delete("test_timestamp_table", new Object[] { "key_0" }); + } + } + + @Test + public void testQueryWithEmptyTable() throws Exception { + // test query with empty table name. + try { + client.insert("", new Object[] { 0L }, new String[] { "c2", "c3" }, new Object[] { + new byte[] { 1 }, "row1" }); + } catch (IllegalArgumentException e) { + Assert.assertEquals("table name is null", ((IllegalArgumentException) e).getMessage()); + } + try { + TableQuery tableQuery = client.query(""); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 250L }); + tableQuery.select("c1", "c2"); + QueryResultSet result = tableQuery.execute(); + } catch (IllegalArgumentException e) { + Assert.assertEquals("table name is null", ((IllegalArgumentException) e).getMessage()); + } + // test insertOrUpdate + final ObTableClient client1 = ObTableClientTestUtil.newTestClient(); + try { + client1.setMetadataRefreshInterval(100); + client1.setServerAddressCachingTimeout(8000); + client1.init(); + long lastTime = getMaxAccessTime(client1); + Thread.sleep(10000); + // test_query_filter_mutate + client1.insertOrUpdate("", "foo", new String[] { "c2" }, new String[] { "bar" }); + long nowTime = getMaxAccessTime(client1); + Assert.assertTrue(nowTime - lastTime > 8000); + } catch (IllegalArgumentException e) { + Assert.assertEquals("table name is null", ((IllegalArgumentException) e).getMessage()); + } + // test async query + try { + TableQuery tableQuery = client.query(""); + tableQuery.addScanRange(new Object[] { 0L }, new Object[] { 250L }); + tableQuery.select("c1", "c2"); + tableQuery.setBatchSize(1); + QueryResultSet result = tableQuery.asyncExecute(); + } catch (IllegalArgumentException e) { + Assert.assertEquals("table name is null", ((IllegalArgumentException) e).getMessage()); + } + } + + @Test + public void testQueryWithScanOrder() throws Exception { + String tableName = "test_query_scan_order"; + ((ObTableClient) client).addRowKeyElement(tableName, new String[] { "c1" }); + try { + client.insert(tableName, new Object[] { 0, 1 }, new String[] { "c3" }, + new Object[] { 2 }); + client.insert(tableName, new Object[] { 0, 2 }, new String[] { "c3" }, + new Object[] { 1 }); + // Forward + Object[] start = { 0, ObObj.getMin() }; + Object[] end = { 1, ObObj.getMax() }; + QueryResultSet resultSet = client.query(tableName).indexName("idx") + .setScanRangeColumns("c1", "c3").addScanRange(start, end).scanOrder(true) + .select("c1", "c2", "c3").execute(); + Assert.assertEquals(2, resultSet.cacheSize()); + int pre_value = 0; + while (resultSet.next()) { + Map valueMap = resultSet.getRow(); + Assert.assertTrue(pre_value < (int) valueMap.get("c3")); + pre_value = (int) valueMap.get("c3"); + } + // Reverse + QueryResultSet resultSet2 = client.query(tableName).indexName("idx") + .setScanRangeColumns("c1", "c3").addScanRange(start, end).scanOrder(false) + .select("c1", "c2", "c3").execute(); + Assert.assertEquals(2, resultSet2.cacheSize()); + pre_value = 3; + while (resultSet2.next()) { + Map valueMap = resultSet2.getRow(); + Assert.assertTrue(pre_value > (int) valueMap.get("c3")); + pre_value = (int) valueMap.get("c3"); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + client.delete(tableName, new Object[] { 0, 1 }); + client.delete(tableName, new Object[] { 0, 2 }); + } + } + + @Test + public void testFirstPartStartEndKeys() throws Exception { + // Get start/end keys of key part + // CREATE TABLE IF NOT EXISTS `testPartitionKeyComplex` ( + // `c0` tinyint NOT NULL, + // `c1` int NOT NULL, + // `c2` bigint NOT NULL, + // `c3` varbinary(1024) NOT NULL, + // `c4` varchar(1024) NOT NULL, + // `c5` varchar(1024) NOT NULL, + // `c6` varchar(20) default NULL, + // PRIMARY KEY (`c0`, `c1`, `c2`, `c3`, `c4`, `c5`) + // ) DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'lz4_1.0' REPLICA_NUM = 3 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 10 + // partition by key(`c0`, `c1`, `c2`, `c3`, `c4`) subpartition by key(`c5`) subpartitions 4 partitions 16; + + ObTableClient tableClient = (ObTableClient) client; + try { + byte[][][] keyFirstPartStartKeys = tableClient + .getFirstPartStartKeys("testPartitionKeyComplex"); + byte[][][] keyFirstPartEndKeys = tableClient + .getFirstPartEndKeys("testPartitionKeyComplex"); + Assert.assertArrayEquals(keyFirstPartStartKeys, keyFirstPartEndKeys); + Assert.assertEquals(1, keyFirstPartStartKeys.length); + Assert.assertEquals(1, keyFirstPartStartKeys[0].length); + Assert.assertEquals(0, keyFirstPartStartKeys[0][0].length); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(true); + } catch (Exception e) { + fail(); + } + } + + @Test + public void testBatchQuery() throws Exception { + String tableName = "test_batch_get"; + final int COUNT_SIZE = 20; + try { + int count = COUNT_SIZE; + // prepare data + for (int i = 0; i < count; i++) { + client.insertOrUpdate(tableName).setRowKey(row(colVal("c1", 1), colVal("c2", i))) + .addMutateRow(row(colVal("c3", i + 1), colVal("c4", i + 2))).execute(); + } + + BatchOperation batchOperation = client.batchOperation(tableName); + while (count-- > 0) { + Row rowKey = row(colVal("c1", 1), colVal("c2", count)); + TableQuery query = null; + if (count % 2 == 0) { + query = query().setRowKey(rowKey); + } else { + query = query().setRowKey(rowKey).select("c2", "c3"); + } + batchOperation.addOperation(query); + } + + BatchOperationResult result = batchOperation.execute(); + int index = COUNT_SIZE; + while (index-- > 0) { + Row row = result.get(index).getOperationRow(); + int c2Val = (int) row.get("c2"); + if (c2Val % 2 == 0) { + assertEquals(4, row.size()); + assertEquals(1, row.get("c1")); + assertEquals(c2Val + 1, row.get("c3")); + assertEquals(c2Val + 2, row.get("c4")); + + } else { + assertEquals(2, row.size()); + assertEquals(c2Val + 1, row.get("c3")); + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + for (int i = 0; i < COUNT_SIZE; i++) { + client.delete(tableName).setRowKey(row(colVal("c1", 1), colVal("c2", i))).execute(); + } + } + } + + @Test + public void test_prefix_scan() throws Exception { + long timeStamp = System.currentTimeMillis(); + String tableName = "test_hbase$fn"; + cleanTable(tableName); + timeStamp = System.currentTimeMillis(); + try { + client.insert(tableName, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value1".getBytes() }); + client.insert(tableName, + new Object[] { "key1_1".getBytes(), "partition".getBytes(), timeStamp + 1 }, + new String[] { "V" }, new Object[] { "value2".getBytes() }); + client.insert(tableName, + new Object[] { "key1_2".getBytes(), "partition".getBytes(), timeStamp }, + new String[] { "V" }, new Object[] { "value3".getBytes() }); + + TableQuery tableQuery = client.query(tableName); + QueryResultSet result = tableQuery.execute(); + Assert.assertEquals(3, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes(), "partition".getBytes()}, + new Object[] { "key1_1".getBytes(), "partition".getBytes()}); + tableQuery.setScanRangeColumns("K", "Q"); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, new Object[] { "key1_2".getBytes() } ); + result = tableQuery.execute(); + Assert.assertEquals(3, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, true, new Object[] { "key1_2".getBytes() }, false ); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, false, new Object[] { "key1_2".getBytes() }, true ); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.setScanRangeColumns("K"); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, false, new Object[] { "key1_2".getBytes() }, false ); + result = tableQuery.execute(); + Assert.assertEquals(0, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.setScanRangeColumns("K"); + tableQuery.indexName("idx_k_v"); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, new Object[] { "key1_2".getBytes() }); + result = tableQuery.execute(); + Assert.assertEquals(3, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.setScanRangeColumns("K"); + tableQuery.indexName("idx_k_v"); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, new Object[] { "key1_1".getBytes() }); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.setScanRangeColumns("K"); + tableQuery.indexName("idx_k_v"); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, false, new Object[] { "key1_2".getBytes() }, true); + result = tableQuery.execute(); + Assert.assertEquals(1, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.setScanRangeColumns("K"); + tableQuery.indexName("idx_k_v"); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, true, new Object[] { "key1_2".getBytes() }, false); + result = tableQuery.execute(); + Assert.assertEquals(2, result.cacheSize()); + + tableQuery = client.query(tableName); + tableQuery.setScanRangeColumns("K"); + tableQuery.indexName("idx_k_v"); + tableQuery.addScanRange(new Object[] { "key1_1".getBytes() }, false, new Object[] { "key1_2".getBytes() }, false); + result = tableQuery.execute(); + Assert.assertEquals(0, result.cacheSize()); + + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + cleanTable(tableName); + } + } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableConnectionTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableConnectionTest.java new file mode 100644 index 00000000..7d673879 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableConnectionTest.java @@ -0,0 +1,173 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.property.Property; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; + +public class ObTableConnectionTest { + public ObTableClient client; + private int connCnt = 100; + private boolean isMannual = false; + + @Before + public void setup() throws Exception { + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.addProperty(Property.SERVER_CONNECTION_POOL_SIZE.getKey(), + Integer.toString(connCnt)); + obTableClient.init(); + + this.client = obTableClient; + } + + int getGvConnections() throws Exception { + int resCnt = 0; + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement + .execute("select count(*) from oceanbase.gv$ob_kv_connections group by svr_ip, svr_port limit 1"); + ResultSet resultSet = statement.getResultSet(); + while (resultSet.next()) { + resCnt = resultSet.getInt(1); + } + assertEquals(false, resultSet.next()); + return resCnt; + } + + int getVConnections() throws Exception { + int resCnt = 0; + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("select count(*) from oceanbase.v$ob_kv_connections"); + ResultSet resultSet = statement.getResultSet(); + while (resultSet.next()) { + resCnt = resultSet.getInt(1); + } + assertEquals(false, resultSet.next()); + return resCnt; + } + + /** + CREATE TABLE `test_varchar_table` ( + `c1` varchar(20) NOT NULL, + `c2` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + ) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = 3 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 10 + **/ + @Test + public void test_conncetion() throws Exception { + String tableName = "test_varchar_table"; + long executeTime = 10000; // 10s + int threadCnt = 10; + ArrayList workers = new ArrayList<>(); + if (client.isOdpMode()) { + Assert.assertTrue(getGvConnections() > 0); + } else { + // when we run all tests, connection may exceed the connCnt + if (isMannual) { + assertEquals(this.connCnt, getGvConnections()); + assertEquals(this.connCnt, getVConnections()); + } else { + Assert.assertTrue(getGvConnections() >= this.connCnt); + Assert.assertTrue(getVConnections() >= this.connCnt); + } + } + + for (int i = 0; i < threadCnt; i++) { + GetWorker worker = new GetWorker(i, tableName, this.client, executeTime); + workers.add(worker); + } + for (int i = 0; i < threadCnt; i++) { + workers.get(i).start(); + } + if (client.isOdpMode()) { + Assert.assertTrue(getGvConnections() > 0); + } else { + // when we run all tests, connection may exceed the connCnt + if (isMannual) { + assertEquals(this.connCnt, getGvConnections()); + assertEquals(this.connCnt, getVConnections()); + } else { + Assert.assertTrue(getGvConnections() >= this.connCnt); + Assert.assertTrue(getVConnections() >= this.connCnt); + } + } + + for (int i = 0; i < threadCnt; i++) { + workers.get(i).join(); + } + if (client.isOdpMode()) { + Assert.assertTrue(getGvConnections() > 0); + } else { + // when we run all tests, connection may exceed the connCnt + if (isMannual) { + assertEquals(this.connCnt, getGvConnections()); + assertEquals(this.connCnt, getVConnections()); + } else { + Assert.assertTrue(getGvConnections() >= this.connCnt); + Assert.assertTrue(getVConnections() >= this.connCnt); + } + } + + this.client.close(); + // when we run all tests, connection may exceed 0 + if (isMannual) { + assertEquals(0, getVConnections()); + assertEquals(0, getGvConnections()); + } + } + + class GetWorker extends Thread { + private int id; + private String tableName; + private ObTableClient obTableClient; + private long executeTime; // in millisecond + + public GetWorker(int id, String tableName, ObTableClient obTableClient, long executeTime) { + this.id = id; + this.tableName = tableName; + this.obTableClient = obTableClient; + this.executeTime = executeTime; + } + + public void run() { + long start = System.currentTimeMillis(); + while ((System.currentTimeMillis() - start) < executeTime) { + try { + obTableClient.get(tableName, new String[] { "k1" }, new String[] { "c1" }); + } catch (Exception e) { + e.printStackTrace(); + System.out.println("thread " + id + " get occurs exception !"); + } finally { + } + } + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableErrMsgTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableErrMsgTest.java new file mode 100644 index 00000000..adc23111 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableErrMsgTest.java @@ -0,0 +1,239 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Before; +import org.junit.Test; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static org.junit.Assert.*; + +public class ObTableErrMsgTest { + ObTableClient client; + public static String tableName = "error_message_table"; + + @Before + public void setup() throws Exception { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { "c1" }); + } + + /** + CREATE TABLE IF NOT EXISTS `error_message_table` ( + `c1` bigint(20) not null, + `c2` varchar(5) not null, + `c3` datetime default current_timestamp, + `c4` varchar(5) generated always as (SUBSTRING(c2, 1)), + `c5` double default 0, + primary key (c1)); + **/ + @Test + public void testTypeNotMatch() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)) // c2 varchar(5) not null + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-10511][OB_KV_COLUMN_TYPE_NOT_MATCH][Column type for 'c2' not match, schema column type is 'VARCHAR', input column type is 'BIGINT']")); + } + + /** + CREATE TABLE IF NOT EXISTS `error_message_table` ( + `c1` bigint(20) not null, + `c2` varchar(5) not null, + `c3` datetime default current_timestamp, + `c4` varchar(5) generated always as (SUBSTRING(c2, 1)), + `c5` double default 0, + primary key (c1)); + **/ + @Test + public void testColumnNotExist() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("cx", 1L)) // cx not exist + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-5217][OB_ERR_BAD_FIELD_ERROR][Unknown column 'cx' in 'error_message_table']")); + } + + /** + CREATE TABLE IF NOT EXISTS `error_message_table` ( + `c1` bigint(20) not null, + `c2` varchar(5) not null, + `c3` datetime default current_timestamp, + `c4` varchar(5) generated always as (SUBSTRING(c2, 1)), + `c5` double default 0, + primary key (c1)); + **/ + @Test + public void testDataTooLong() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", "123456")) // c2 varchar(5) not null + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-5167][OB_ERR_DATA_TOO_LONG][Data too long for column 'c2' at row 0]")); + } + + /** + CREATE TABLE IF NOT EXISTS `error_message_table` ( + `c1` bigint(20) not null, + `c2` varchar(5) not null, + `c3` datetime default current_timestamp, + `c4` varchar(5) generated always as (SUBSTRING(c2, 1)), + `c5` double default 0, + primary key (c1)); + **/ + @Test + public void testDataOverflow() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + String dateString = "2023-04-05 14:30:00.123"; + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + Date date = sdf.parse(dateString); + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", "a")) + .addMutateColVal(colVal("c3", date)) // `c3` datetime default current_timestamp, + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4157][OB_OPERATE_OVERFLOW][DATETIME value is out of range in 'c3']")); + } + + /** + CREATE TABLE IF NOT EXISTS `error_message_table` ( + `c1` bigint(20) not null, + `c2` varchar(5) not null, + `c3` datetime default current_timestamp, + `c4` varchar(5) generated always as (SUBSTRING(c2, 1)), + `c5` double default 0, + primary key (c1)); + **/ + @Test + public void testRowKeyCountNotMatch() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.insert(tableName).setRowKey(colVal("c1", 1L), colVal("c2", 1L)) + .addMutateColVal(colVal("c2", "a")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-10510][OB_KV_ROWKEY_COUNT_NOT_MATCH][Rowkey column count not match, schema rowkey count is '1', input rowkey count is '2']")); + } + + /** + CREATE TABLE IF NOT EXISTS `error_message_table` ( + `c1` bigint(20) not null, + `c2` varchar(5) not null, + `c3` datetime default current_timestamp, + `c4` varchar(5) generated always as (SUBSTRING(c2, 1)), + `c5` double default 0, + primary key (c1)); + **/ + @Test + public void testMutateRowKeyNotSupport() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.insert(tableName, 1L, new String[] {"c1", "c2"}, new Object[]{2L, "a"}); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][mutate rowkey column not supported]")); + } + + /** + CREATE TABLE IF NOT EXISTS `error_message_table` ( + `c1` bigint(20) not null, + `c2` varchar(5) not null, + `c3` datetime default current_timestamp, + `c4` varchar(5) generated always as (SUBSTRING(c2, 1)), + `c5` double default 0, + primary key (c1)); + **/ + @Test + public void testPrimaryKeyDuplicate() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", "a")) + .execute(); + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", "a")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-5024][OB_ERR_PRIMARY_KEY_DUPLICATE][Duplicate entry '1' for key 'PRIMARY']")); + } + + /** + CREATE TABLE IF NOT EXISTS `error_message_table` ( + `c1` bigint(20) not null, + `c2` varchar(5) not null, + `c3` datetime default current_timestamp, + `c4` varchar(5) generated always as (SUBSTRING(c2, 1)), + `c5` double default 0, + primary key (c1)); + **/ + @Test + public void testUpdateVirtualColumnNotSupport() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.update(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c4", "a")) // c4 varchar(5) generated always as (substr(c2, 1)), + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][The specified value for generated column not supported]")); + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableGlobalIndexTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableGlobalIndexTest.java new file mode 100644 index 00000000..c8711598 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableGlobalIndexTest.java @@ -0,0 +1,583 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2022 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.mutation.Row; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.Map; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static org.junit.Assert.assertEquals; + +public class ObTableGlobalIndexTest { + ObTableClient client; + + @Before + public void setup() throws Exception { + setEnableIndexDirectSelect(); + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + } + + public void setEnableIndexDirectSelect() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("set global ob_enable_index_direct_select = on"); + } + + public void cleanPartitionLocationTable(String tableName) throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("delete from " + tableName); + } + + public void checkIndexTableRow(String tableName, int recordCount) throws Exception { + String sql1 = "select table_name from oceanbase.__all_virtual_table where data_table_id = " + + "(select table_id from oceanbase.__all_virtual_table where table_name = '" + + tableName + "')"; + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql1); + resultSet.next(); + String indexTableName = resultSet.getString(1); + Statement statement2 = connection.createStatement(); + ResultSet resultSet2 = statement2.executeQuery("select count(*) from " + indexTableName); + resultSet2.next(); + Assert.assertEquals(recordCount, resultSet2.getLong(1)); + } + + public int gen_key(int i) { + int key = i; + if (i % 3 == 0) { + key = i; + } else if (i % 3 == 1) { + key = i + 100 + 1; + } else if (i % 3 == 2) { + key = i + 200 + 2; + } + return key; + } + + @Test + public void test_insert() throws Exception { + test_insert("test_global_hash_hash", 20); + test_insert("test_global_key_key", 20); + test_insert("test_global_range_range", 20); + } + + public void test_insert(String tableName, int recordCount) throws Exception { + try { + client.addRowKeyElement(tableName, new String[] { "C1" }); + String[] properties_name = { "C2", "C3" }; + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.insert(tableName, new Object[] { key }, properties_name, + new Object[] { key + 1, ("hello " + key).getBytes() }); + Assert.assertEquals(1, affectRows); + } + // check index table row counts + checkIndexTableRow(tableName, recordCount); + + // get data + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + Map result = client.get(tableName, new Object[] { key }, + properties_name); + Assert.assertEquals(key + 1, result.get("C2")); + Assert.assertEquals("hello " + key, result.get("C3")); + } + + } finally { + // delete data + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.delete(tableName, new Object[] { key }); + Assert.assertEquals(1, affectRows); + } + checkIndexTableRow(tableName, 0); + } + } + + @Test + public void test_update() throws Exception { + test_update("test_global_hash_hash", 20); + test_update("test_global_key_key", 20); + test_update("test_global_range_range", 20); + } + + public void test_update(String tableName, int recordCount) throws Exception { + try { + client.addRowKeyElement(tableName, new String[] { "C1" }); + String[] properties_name = { "C2", "C3" }; + // prepare data + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.insert(tableName, new Object[] { key }, properties_name, + new Object[] { key + 1, ("hello " + key).getBytes() }); + Assert.assertEquals(1, affectRows); + } + // update global index key + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.update(tableName, new Object[] { key }, + new String[] { "C2" }, new Object[] { key + 2 }); + Assert.assertEquals(1, affectRows); + Map result = client.get(tableName, new Object[] { key }, + properties_name); + Assert.assertEquals(key + 2, result.get("C2")); + Assert.assertEquals("hello " + key, result.get("C3")); + } + // update other key + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.update(tableName, new Object[] { key }, + new String[] { "C3" }, new Object[] { "hi " + key }); + Assert.assertEquals(1, affectRows); + Map result = client.get(tableName, new Object[] { key }, + properties_name); + Assert.assertEquals(key + 2, result.get("C2")); + Assert.assertEquals("hi " + key, result.get("C3")); + } + checkIndexTableRow(tableName, recordCount); + } finally { + // delete data + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.delete(tableName, new Object[] { key }); + Assert.assertEquals(1, affectRows); + } + checkIndexTableRow(tableName, 0); + } + } + + @Test + public void test_insert_or_update() throws Exception { + test_insert_or_update("test_global_hash_hash", 20); + test_insert_or_update("test_global_key_key", 20); + test_insert_or_update("test_global_range_range", 20); + } + + public void test_insert_or_update(String tableName, int recordCount) throws Exception { + try { + client.addRowKeyElement(tableName, new String[] { "C1" }); + String[] properties_name = { "C2", "C3" }; + // prepare data: insert + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.insertOrUpdate(tableName, new Object[] { key }, + properties_name, new Object[] { key + 1, ("hello " + key).getBytes() }); + Assert.assertEquals(1, affectRows); + Map result = client.get(tableName, new Object[] { key }, + properties_name); + Assert.assertEquals(key + 1, result.get("C2")); + Assert.assertEquals("hello " + key, result.get("C3")); + } + + // insert again: update + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.insertOrUpdate(tableName, new Object[] { key }, + properties_name, new Object[] { key + 2, ("hi " + key).getBytes() }); + Assert.assertEquals(1, affectRows); + Map result = client.get(tableName, new Object[] { key }, + properties_name); + Assert.assertEquals(key + 2, result.get("C2")); + Assert.assertEquals("hi " + key, result.get("C3")); + } + + checkIndexTableRow(tableName, recordCount); + } finally { + // delete data + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.delete(tableName, new Object[] { key }); + Assert.assertEquals(1, affectRows); + } + checkIndexTableRow(tableName, 0); + } + } + + @Test + public void test_replace() throws Exception { + test_replace("test_global_hash_hash", 20); + test_replace("test_global_key_key", 20); + test_replace("test_global_range_range", 20); + } + + public void test_replace(String tableName, int recordCount) throws Exception { + try { + client.addRowKeyElement(tableName, new String[] { "C1" }); + String[] properties_name = { "C2", "C3" }; + // prepare data: insert + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.replace(tableName, new Object[] { key }, properties_name, + new Object[] { key + 1, ("hello " + key).getBytes() }); + Assert.assertEquals(1, affectRows); + Map result = client.get(tableName, new Object[] { key }, + properties_name); + Assert.assertEquals(key + 1, result.get("C2")); + Assert.assertEquals("hello " + key, result.get("C3")); + } + + // insert again: update + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.replace(tableName, new Object[] { key }, properties_name, + new Object[] { key + 2, ("hi " + key).getBytes() }); + Assert.assertEquals(2, affectRows); + Map result = client.get(tableName, new Object[] { key }, + properties_name); + Assert.assertEquals(key + 2, result.get("C2")); + Assert.assertEquals("hi " + key, result.get("C3")); + } + + checkIndexTableRow(tableName, recordCount); + } finally { + // delete data + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.delete(tableName, new Object[] { key }); + Assert.assertEquals(1, affectRows); + } + checkIndexTableRow(tableName, 0); + } + } + + @Test + public void test_increment_append() throws Exception { + test_increment_append("test_global_hash_hash", 20); + test_increment_append("test_global_key_key", 20); + test_increment_append("test_global_range_range", 20); + + } + + public void test_increment_append(String tableName, int recordCount) throws Exception { + try { + client.addRowKeyElement(tableName, new String[] { "c1" }); + String[] properties_name = { "c2" }; + // increment without record + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + Map affect_res = client.increment(tableName, new Object[] { key }, + properties_name, new Object[] { key + 1 }, true); + Assert.assertEquals(key + 1, affect_res.get("c2")); + Map get_result = client.get(tableName, new Object[] { key }, + properties_name); + Assert.assertEquals(key + 1, get_result.get("c2")); + } + + // increment without column value + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + Map affect_res = client.increment(tableName, new Object[] { key }, + properties_name, new Object[] { key + 1 }, true); + Assert.assertEquals(2 * key + 2, affect_res.get("c2")); + Map get_result = client.get(tableName, new Object[] { key }, + properties_name); + Assert.assertEquals(2 * key + 2, get_result.get("c2")); + } + + // append with empty column value + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + Map affect_res = client.append(tableName, new Object[] { key }, + new String[] { "c3" }, new Object[] { "hi~".getBytes() }, true); + Assert.assertEquals("hi~", affect_res.get("c3")); + Map get_result = client.get(tableName, new Object[] { key }, + new String[] { "c3" }); + Assert.assertEquals("hi~", get_result.get("c3")); + } + checkIndexTableRow(tableName, recordCount); + + // append with not empty column value + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + Map affect_res = client.append(tableName, new Object[] { key }, + new String[] { "c3" }, new Object[] { " hi~".getBytes() }, true); + Assert.assertEquals("hi~ hi~", affect_res.get("c3")); + Map get_result = client.get(tableName, new Object[] { key }, + new String[] { "c3" }); + Assert.assertEquals("hi~ hi~", get_result.get("c3")); + } + checkIndexTableRow(tableName, recordCount); + } finally { + // delete data + for (int i = 0; i < recordCount; i++) { + int key = gen_key(i); + long affectRows = client.delete(tableName, new Object[] { key }); + Assert.assertEquals(1, affectRows); + } + checkIndexTableRow(tableName, 0); + } + } + + @Test + public void test_query_hash_range() throws Exception { + String tableName = "test_global_hash_range"; + test_query_in_global_index_table(tableName); + } + + @Test + public void test_query_in_local_index_not_use_rowkey() throws Exception { + Assert.assertThrows(IllegalArgumentException.class, () -> { + test_query_in_global_index_table("test_global_index_no_part"); + }); + } + + @Test + public void test_non_partition_index_table() throws Exception { + test_query_in_global_index_table("test_global_all_no_part"); + test_query_in_global_index_table("test_global_primary_no_part"); + } + + public void test_query_in_global_index_table(String tableName) throws Exception { + try { + client.addRowKeyElement(tableName, new String[] { "C1" }); + // prepare data + int recordCount = 20; + String[] properties_name = { "C2", "C3" }; + Object[] properties_value = null; + for (int i = 0; i < recordCount; i++) { + if (i % 3 == 0) { + properties_value = new Object[] { i + 1, i + 2 }; + } else if (i % 3 == 1) { + properties_value = new Object[] { i + 100 + 1, i + 100 + 2 }; + } else if (i % 3 == 2) { + properties_value = new Object[] { i + 200 + 1, i + 200 + 2 }; + } + long affectRows = client.insert(tableName, new Object[] { i }, properties_name, + properties_value); + Assert.assertEquals(1, affectRows); + } + + // query by primary index + TableQuery query = client.query(tableName); + query.addScanRange(new Object[] { 0 }, new Object[] { recordCount }); + + QueryResultSet resultSet = query.execute(); + int count = 0; + while (resultSet.next()) { + Map row = resultSet.getRow(); + int c1 = (int) row.get("C1"); + int c2 = (int) row.get("C2"); + int c3 = (int) row.get("C3"); + if (c1 % 3 == 0) { + Assert.assertEquals(c1 + 1, c2); + Assert.assertEquals(c1 + 2, c3); + } else if (c1 % 3 == 1) { + Assert.assertEquals(c1 + 100 + 1, c2); + Assert.assertEquals(c1 + 100 + 2, c3); + } else if (c1 % 3 == 2) { + Assert.assertEquals(c1 + 200 + 1, c2); + Assert.assertEquals(c1 + 200 + 2, c3); + } + count++; + } + Assert.assertEquals(count, recordCount); + + // query by global index, will lookup primary table + TableQuery query2 = client.query(tableName).indexName("idx"); + query2.setScanRangeColumns("C2"); + query2.addScanRange(new Object[] { 0 }, new Object[] { recordCount + 200 + 1 }); + QueryResultSet resultSet2 = query2.execute(); + count = 0; + while (resultSet2.next()) { + Map row = resultSet2.getRow(); + int c1 = (int) row.get("C1"); + int c2 = (int) row.get("C2"); + int c3 = (int) row.get("C3"); + if (c1 % 3 == 0) { + Assert.assertEquals(c1 + 1, c2); + Assert.assertEquals(c1 + 2, c3); + } else if (c1 % 3 == 1) { + Assert.assertEquals(c1 + 100 + 1, c2); + Assert.assertEquals(c1 + 100 + 2, c3); + } else if (c1 % 3 == 2) { + Assert.assertEquals(c1 + 200 + 1, c2); + Assert.assertEquals(c1 + 200 + 2, c3); + } + count++; + } + + // query by global index, without lookup primary table + TableQuery query3 = client.query(tableName).indexName("idx"); + query3.setScanRangeColumns("C2"); + query3.addScanRange(new Object[] { 0 }, new Object[] { recordCount + 200 + 1 }); + query3.select("C2"); + QueryResultSet resultSet3 = query3.execute(); + Assert.assertEquals(resultSet3.cacheSize(), recordCount); + + // query by local index, will lookup primary table + TableQuery query4 = client.query(tableName).indexName("idx2"); + query4.addScanRange(new Object[] { 0 }, new Object[] { recordCount + 200 + 2 }); + query4.select("C1", "C2", "C3"); + QueryResultSet resultSet4 = query4.execute(); + Assert.assertEquals(resultSet4.cacheSize(), recordCount); + count = 0; + while (resultSet4.next()) { + Map row = resultSet4.getRow(); + int c1 = (int) row.get("C1"); + int c2 = (int) row.get("C2"); + int c3 = (int) row.get("C3"); + if (c1 % 3 == 0) { + Assert.assertEquals(c1 + 1, c2); + Assert.assertEquals(c1 + 2, c3); + } else if (c1 % 3 == 1) { + Assert.assertEquals(c1 + 100 + 1, c2); + Assert.assertEquals(c1 + 100 + 2, c3); + } else if (c1 % 3 == 2) { + Assert.assertEquals(c1 + 200 + 1, c2); + Assert.assertEquals(c1 + 200 + 2, c3); + } + count++; + } + + } finally { + cleanPartitionLocationTable(tableName); + } + } + + @Test + public void test_query_async() throws Exception { + String tableName = "test_global_hash_range"; + test_query_async(tableName); + } + + public void test_query_async(String tableName) throws Exception { + try { + client.addRowKeyElement(tableName, new String[] { "C1" }); + // prepare data + int recordCount = 20; + String[] properties_name = { "C2", "C3" }; + Object[] properties_value = null; + for (int i = 0; i < recordCount; i++) { + if (i % 3 == 0) { + properties_value = new Object[] { i + 1, i + 2 }; + } else if (i % 3 == 1) { + properties_value = new Object[] { i + 100 + 1, i + 100 + 2 }; + } else if (i % 3 == 2) { + properties_value = new Object[] { i + 200 + 1, i + 200 + 2 }; + } + long affectRows = client.insert(tableName, new Object[] { i }, properties_name, + properties_value); + Assert.assertEquals(1, affectRows); + } + + // query sync by global index + TableQuery query = client.query(tableName).indexName("idx"); + query.setScanRangeColumns("C2"); + query.addScanRange(new Object[] { 0 }, new Object[] { recordCount + 200 + 1 }); + query.select("C1", "C2", "C3"); + query.limit(5); + query.setBatchSize(2); + query.setMaxResultSize(10000); + QueryResultSet result = query.asyncExecute(); + for (int i = 0; i < 5; i++) { + Assert.assertTrue(result.next()); + Map row = result.getRow(); + int c1 = (int) row.get("C1"); + int c2 = (int) row.get("C2"); + int c3 = (int) row.get("C3"); + if (c1 % 3 == 0) { + Assert.assertEquals(c1 + 1, c2); + Assert.assertEquals(c1 + 2, c3); + } else if (c1 % 3 == 1) { + Assert.assertEquals(c1 + 100 + 1, c2); + Assert.assertEquals(c1 + 100 + 2, c3); + } else if (c1 % 3 == 2) { + Assert.assertEquals(c1 + 200 + 1, c2); + Assert.assertEquals(c1 + 200 + 2, c3); + } + } + } finally { + cleanPartitionLocationTable(tableName); + } + } + + /** + CREATE TABLE `test_ttl_timestamp_with_index` ( + `c1` varchar(20) NOT NULL, + `c2` bigint NOT NULL, + `c3` bigint DEFAULT NULL, + `c4` bigint DEFAULT NULL, + `expired_ts` timestamp, + PRIMARY KEY (`c1`, `c2`), + KEY `idx`(`c1`, `c4`) local, + KEY `idx2`(`c3`) global partition by hash(`c3`) partitions 4) TTL(expired_ts + INTERVAL 0 SECOND) partition by key(`c1`) partitions 4; + **/ + @Test + public void test_ttl_query_with_global_index() throws Exception { + String tableName = "test_ttl_timestamp_with_index"; + String rowKey1 = "c1"; + String rowKey2 = "c2"; + String intCol = "c3"; + String intCol2 = "c4"; + String expireCol = "expired_ts"; + String prefixKey = "test"; + long[] keyIds = { 1L, 2L }; + try { + // 1. insert records with null expired_ts + for (long id : keyIds) { + client.insert(tableName).setRowKey(colVal(rowKey1, prefixKey), colVal(rowKey2, id)) + .addMutateColVal(colVal(intCol, id + 100)) + .addMutateColVal(colVal(intCol2, id + 200)) + .addMutateColVal(colVal(expireCol, null)).execute(); + } + // 2. query all inserted records + QueryResultSet resultSet = client.query(tableName).indexName("idx2") + .setScanRangeColumns(intCol) + .addScanRange(new Object[] { 101L }, new Object[] { 102L }).execute(); + Assert.assertEquals(resultSet.cacheSize(), keyIds.length); + + // 3. update the expired_ts + Timestamp curTs = new Timestamp(System.currentTimeMillis()); + client.update(tableName) + .setRowKey(colVal(rowKey1, prefixKey), colVal(rowKey2, keyIds[1])) + .addMutateColVal(colVal(expireCol, curTs)).execute(); + + // 3. re-query all inserted records, the expired record won't be returned + resultSet = client.query(tableName).indexName("idx2").setScanRangeColumns(intCol) + .addScanRange(new Object[] { 101L }, new Object[] { 102L }).execute(); + Assert.assertEquals(resultSet.cacheSize(), 1); + Assert.assertTrue(resultSet.next()); + Row row = resultSet.getResultRow(); + Assert.assertEquals(row.get(rowKey2), keyIds[0]); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (long id : keyIds) { + client.delete(tableName).setRowKey(colVal(rowKey1, prefixKey), colVal(rowKey2, id)) + .execute(); + } + } + } + +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableHotkeyThrottleTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableHotkeyThrottleTest.java new file mode 100644 index 00000000..d2ec64d3 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableHotkeyThrottleTest.java @@ -0,0 +1,94 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2022 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.util.ObTableHotkeyThrottleUtil; +import org.junit.Test; + +import java.util.List; +import java.util.ArrayList; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; + +public class ObTableHotkeyThrottleTest { + public int threadNum = 32; + public int repeatKeyNum = 4; + public long startTime; + + @Test + public void testSimpleThrottle() throws Exception { + try { + List allWorkers = new ArrayList<>(); + startTime = System.currentTimeMillis(); + for (int i = 0; i < threadNum; ++i) { + ObTableHotkeyThrottleUtil worker = new ObTableHotkeyThrottleUtil(); + if (i < threadNum/4) { + worker.init(threadNum, i, startTime, "test_throttle", new String[] { "c1", "c2" }, + ObTableHotkeyThrottleUtil.TestType.random, ObTableHotkeyThrottleUtil.OperationType.insertOrUpdate, 800, null, 0); + } else if (i < threadNum/3) { + worker.init(threadNum, i, startTime, "test_throttle", new String[] { "c1", "c2" }, ObTableHotkeyThrottleUtil.TestType.random, ObTableHotkeyThrottleUtil.OperationType.query, 800, null, 0); + } else { + long rowKeyNum = i % repeatKeyNum; + String rowkeyString = "Test" + rowKeyNum; + worker.init(threadNum, i, startTime, "test_throttle", new String[] { "c1", "c2" }, + ObTableHotkeyThrottleUtil.TestType.specifiedKey, ObTableHotkeyThrottleUtil.OperationType.query, 800, + null, 0, colVal("c1", rowKeyNum), colVal("c2", rowkeyString)); + } + allWorkers.add(worker); + worker.start(); + } + + // for each 2s, get unitOperationTimes and unitOperationTimes from ObTableHotkeyThrottleUtil and print + for (int i = 0; i < 10; ++i) { + Thread.sleep(2000); + List blockList = allWorkers.get(0).getUnitBlockTimes(); + List operationList = allWorkers.get(0).getUnitOperationTimes(); + System.out.print("Block times: "); + for (int j = 0; j < threadNum; ++j) { + System.out.print(blockList.get(j) + " "); + } + System.out.print("\nOperation times: "); + for (int j = 0; j < threadNum; ++j) { + System.out.print(operationList.get(j) + " "); + } + System.out.print("\n"); + } + + for (int i = 0; i < threadNum; ++i) { + allWorkers.get(i).join(); + System.out.println("Thread " + i + "has finished"); + } + + // get totalBlockTimes and totalOperationTimes from ObTableHotkeyThrottleUtil and print + List blockList = allWorkers.get(0).getTotalBlockTimes(); + List operationList = allWorkers.get(0).getTotalOperationTimes(); + System.out.print("Total Block times: "); + for (int j = 0; j < threadNum; ++j) { + System.out.print(blockList.get(j) + " "); + } + System.out.print("\nTotal Operation times: "); + for (int j = 0; j < threadNum; ++j) { + System.out.print(operationList.get(j) + " "); + } + System.out.print("\n"); + + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableIndexWithCalcColumn.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableIndexWithCalcColumn.java new file mode 100644 index 00000000..5b253b15 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableIndexWithCalcColumn.java @@ -0,0 +1,633 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2022 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.mutation.Row; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.Map; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.row; +import static org.junit.Assert.*; + +public class ObTableIndexWithCalcColumn { + + String CreateTableStatement = "CREATE TABLE `index_has_current_timestamp` (\n" + + " `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" + + " `adiu` varchar(512) NOT NULL DEFAULT '',\n" + + " `mode` varchar(512) NOT NULL DEFAULT '',\n" + + " `time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n" + + " `tag` varchar(512) DEFAULT '',\n" + + " `content` varchar(412) DEFAULT '',\n" + + " PRIMARY KEY (`id`, `adiu`),\n" + + " KEY `idx_adiu_time_mode_tag` (`id`, `adiu`, `time`, `mode`) BLOCK_SIZE 16384 LOCAL,\n" + + " KEY `g_idx_time_tag_mode` (`time`, `tag`, `mode`) BLOCK_SIZE 16384 GLOBAL\n" + + " ) TTL(time + INTERVAL 300 second) partition by key(adiu) partitions 8;"; + + String TableName = "index_has_current_timestamp"; + Long TableId; + String LocalIndexTableName; + String GlobalIndexTableName; + String StringFormat = "%s_%d"; + String[] AllColumns = { "id", "adiu", "mode", "time", "tag", "content" }; + int recordCount = 10; + ObTableClient client; + + @Before + public void setup() throws Exception { + setEnableIndexDirectSelect(); + createTable(); + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + this.client.addRowKeyElement(TableName, new String[] { "id", "adiu" }); + } + + @After + public void teardown() throws Exception { + dropTable(); + } + + public void setEnableIndexDirectSelect() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("set global ob_enable_index_direct_select = on"); + } + + public void createTable() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute(CreateTableStatement); + ResultSet rs = statement + .executeQuery("select table_id from oceanbase.__all_table where table_name = '" + + TableName + "'"); + if (rs.next()) { + TableId = rs.getLong(1); + LocalIndexTableName = "__idx_" + TableId + "_idx_adiu_time_mode_tag"; + GlobalIndexTableName = "__idx_" + TableId + "_g_idx_time_tag_mode"; + } + statement.close(); + } + + public void dropTable() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("drop table " + TableName); + } + + public void deleteTable() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("delete from " + TableName); + } + + public void removeTTLAttribute() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("alter table " + TableName + " remove TTL"); + } + + public void addTTLAttribute(int expire_secord) throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("alter table " + TableName + " TTL (time + INTERVAL " + expire_secord + + " SECOND)"); + } + + private void checkIndexData(long count) throws Exception { + String sql = "select count(1) as cnt from " + LocalIndexTableName; + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql); + if (resultSet.next()) { + long total = resultSet.getLong(1); + Assert.assertEquals(count, total); + } else { + Assert.fail("there is no data for " + LocalIndexTableName); + } + sql = "select count(1) as cnt from " + GlobalIndexTableName; + resultSet = statement.executeQuery(sql); + if (resultSet.next()) { + long total = resultSet.getLong(1); + Assert.assertEquals(count, total); + } else { + Assert.fail("there is no data for " + GlobalIndexTableName); + } + statement.close(); + } + + @Test + public void test_without_ttl_attributes() throws Exception { + recordCount = 10; + removeTTLAttribute(); + test_insert(); + test_update(); + test_insert_up(); + test_replace(); + test_delete(); + test_query(); + } + + @Test + public void test_with_ttl_attribute() throws Exception { + recordCount = 10; + // test rows has been not expired + addTTLAttribute(300); + test_insert(); + test_update(); + test_insert_up(); + test_replace(); + test_delete(); + test_query(); + // test rows has been expired + addTTLAttribute(5); + test_insert_with_expired_row(); + test_query_with_expired_row(); + } + + public void test_insert_with_expired_row() throws Exception { + try { + insert("insert", recordCount, true); + Thread.sleep(5000); + update("insertOrupdate", recordCount, true); + checkIndexData(recordCount); + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTable(); + } + } + + public void insert(String op_type, int count, boolean fill_autoinc) throws Exception { + for (int i = 1; i <= count; i++) { + String adiu = String.format(StringFormat, "adiu", i); + String mode = String.format(StringFormat, "mode", i); + String tag = String.format(StringFormat, "tag", i); + String content = String.format(StringFormat, "content", i); + Object autoincObj = fill_autoinc ? Long.valueOf(i) : null; + Row rowKey = row(colVal("id", autoincObj), colVal("adiu", adiu)); + Row row = row(); + row.add(colVal("mode", mode)); + row.add(colVal("tag", tag)); + row.add(colVal("content", content)); + if ("insert".equalsIgnoreCase(op_type)) { + client.insert(TableName).setRowKey(rowKey).addMutateRow(row).execute(); + } + if ("insertOrUpdate".equalsIgnoreCase(op_type)) { + client.insertOrUpdate(TableName).setRowKey(rowKey).addMutateRow(row).execute(); + } + if ("replace".equalsIgnoreCase(op_type)) { + client.replace(TableName).setRowKey(rowKey).addMutateRow(row).execute(); + } + } + } + + public void update(String op_type, int count, boolean is_expired) throws Exception { + for (int i = 1; i <= count; i++) { + Long id = Long.valueOf(i); + String adiu = String.format(StringFormat, "adiu", i); + Map valueMap = client.get(TableName, new Object[] { id, adiu }, + AllColumns); + Timestamp time1 = new Timestamp(System.currentTimeMillis() + 100000); + if (is_expired) { + Assert.assertTrue(valueMap.isEmpty()); + } else { + Assert.assertEquals(id, valueMap.get("id")); + Assert.assertEquals(String.format(StringFormat, "adiu", id), valueMap.get("adiu")); + time1 = (Timestamp) valueMap.get("time"); + } + + // do update + Row rowKey = row(colVal("id", id), colVal("adiu", adiu)); + Row row = row(); + String update_mode = String.format(StringFormat, "mode_update", i); + String update_tag = String.format(StringFormat, "mode_tag", i); + String update_content = String.format(StringFormat, "mode_content", i); + row.add(colVal("mode", update_mode)); + row.add(colVal("tag", update_tag)); + row.add(colVal("content", update_content)); + if ("update".equalsIgnoreCase(op_type)) { + client.update(TableName).setRowKey(rowKey).addMutateRow(row).execute(); + } + if ("insertOrUpdate".equalsIgnoreCase(op_type)) { + client.insertOrUpdate(TableName).setRowKey(rowKey).addMutateRow(row).execute(); + } + if ("replace".equalsIgnoreCase(op_type)) { + client.replace(TableName).setRowKey(rowKey).addMutateRow(row).execute(); + } + // get again + Map valueMap_2 = client.get(TableName, new Object[] { id, adiu }, + AllColumns); + Assert.assertEquals(id, valueMap_2.get("id")); + Assert.assertEquals(String.format(StringFormat, "adiu", id), valueMap_2.get("adiu")); + Assert.assertEquals(update_mode, valueMap_2.get("mode")); + Assert.assertEquals(update_tag, valueMap_2.get("tag")); + Assert.assertEquals(update_content, valueMap_2.get("content")); + if (is_expired) { + Assert.assertTrue(time1.after((Timestamp) valueMap_2.get("time"))); + } else { + Assert.assertTrue(time1.before((Timestamp) valueMap_2.get("time"))); + } + } + } + + public void test_insert() throws Exception { + try { + insert("insert", recordCount, false); + for (int i = 1; i <= recordCount; i++) { + Long id = Long.valueOf(i); + String adiu = String.format(StringFormat, "adiu", i); + Map valueMap = client.get(TableName, new Object[] { id, adiu }, + AllColumns); + Assert.assertEquals(id, valueMap.get("id")); + Assert.assertEquals(String.format(StringFormat, "adiu", id), valueMap.get("adiu")); + } + checkIndexData(recordCount); + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTable(); + } + } + + public void test_update() throws Exception { + try { + insert("insert", recordCount, true); + Thread.sleep(1000); + update("update", recordCount, false); + checkIndexData(recordCount); + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTable(); + } + + } + + public void test_insert_up() throws Exception { + try { + insert("insertOrUpdate", recordCount, true); + Thread.sleep(1000); + update("insertOrUpdate", recordCount, false); + checkIndexData(recordCount); + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTable(); + } + } + + public void test_replace() throws Exception { + try { + insert("replace", recordCount, true); + Thread.sleep(1000); + update("replace", recordCount, false); + checkIndexData(recordCount); + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTable(); + } + } + + public void test_delete() throws Exception { + try { + insert("insert", recordCount, true); + checkIndexData(recordCount); + for (int i = 1; i <= recordCount; i++) { + Long id = Long.valueOf(i); + String adiu = String.format(StringFormat, "adiu", i); + Map valueMap = client.get(TableName, new Object[] { id, adiu }, + AllColumns); + Assert.assertEquals(id, valueMap.get("id")); + Assert.assertEquals(String.format(StringFormat, "adiu", id), valueMap.get("adiu")); + Row rowKey = row(colVal("id", id), colVal("adiu", adiu)); + client.delete(TableName).setRowKey(rowKey).execute(); + } + checkIndexData(0); + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTable(); + } + } + + public void test_query() throws Exception { + try { + insert("insert", recordCount, true); + test_query(false); + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTable(); + } + } + + public void test_query_with_expired_row() throws Exception { + try { + insert("insert", recordCount, true); + Thread.sleep(5000); + test_query(true); + } catch (Exception e) { + e.printStackTrace(); + } finally { + deleteTable(); + } + } + + public void test_query(boolean is_expire) throws Exception { + // query with primary index + String start_adiu = String.format(StringFormat, "adiu", 1); + Object[] start = { 0L, start_adiu }; + String end_adiu = String.format(StringFormat, "adiu", recordCount); + Object[] end = { Long.valueOf(recordCount), end_adiu }; + QueryResultSet resultSet = client.query(TableName).addScanRange(start, end).execute(); + if (is_expire) { + Assert.assertEquals(resultSet.cacheSize(), 0); + } else { + Assert.assertEquals(resultSet.cacheSize(), recordCount); + } + + // query with local index + Timestamp start_time1 = new Timestamp(System.currentTimeMillis() - 100000); + start_time1.setNanos(0); + String start_adiu1 = String.format(StringFormat, "adiu", 1); + String start_mode1 = String.format(StringFormat, "mode", 1); + Object[] start1 = { 0L, start_adiu1, start_time1, start_mode1 }; + Timestamp end_time1 = new Timestamp(System.currentTimeMillis() + 100000); + end_time1.setNanos(0); + String end_adiu1 = String.format(StringFormat, "adiu", recordCount); + String end_mode1 = String.format(StringFormat, "mode", recordCount); + Object[] end1 = { Long.valueOf(recordCount), end_adiu1, end_time1, end_mode1 }; + QueryResultSet resultSet1 = client.query(TableName).indexName("idx_adiu_time_mode_tag") + .setScanRangeColumns("id", "time", "tag", "mode").addScanRange(start1, end1).execute(); + if (is_expire) { + Assert.assertEquals(resultSet1.cacheSize(), 0); + } else { + Assert.assertEquals(resultSet1.cacheSize(), recordCount); + } + // query with global index + Timestamp startTime2 = new Timestamp(System.currentTimeMillis() - 100000); + startTime2.setNanos(0); + String start_tag = String.format(StringFormat, "tag", 1); + String start_mode = String.format(StringFormat, "mode", 1); + Object[] start2 = { startTime2, start_tag, start_mode }; + Timestamp endTime2 = new Timestamp(System.currentTimeMillis() + 100000); + endTime2.setNanos(0); + String end_tag = String.format(StringFormat, "tag", recordCount); + String end_mode = String.format(StringFormat, "mode", recordCount); + Object[] end2 = { endTime2, end_tag, end_mode }; + QueryResultSet resultSet2 = client.query(TableName).indexName("g_idx_time_tag_mode") + .setScanRangeColumns("time", "tag", "mode").addScanRange(start2, end2).execute(); + if (is_expire) { + Assert.assertEquals(resultSet2.cacheSize(), 0); + } else { + Assert.assertEquals(resultSet2.cacheSize(), recordCount); + } + } + + @Test + public void testLocalIndexHasGenerateColumn() throws Exception { + /* + CREATE TABLE `test_local_index_with_vgen_col` ( + `name` varchar(512) NOT NULL DEFAULT '', + `pk` varchar(512) NOT NULL, + `adiu` varchar(512) NOT NULL DEFAULT '', + `id` bigint(20) NOT NULL DEFAULT 0, + `name_v` varchar(20) GENERATED ALWAYS AS (substr(`name`,1,5)) VIRTUAL, + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`adiu`, `pk`, `gmt_create`), + KEY `idx_adiu_v_name` (`adiu`, `name_v`) BLOCK_SIZE 16384 LOCAL + ) TTL (gmt_create + INTERVAL 300 SECOND) partition by key(adiu) partitions 8; + */ + String TABLE_NAME = "test_local_index_with_vgen_col"; + testIndexHasGenerateColumn(TABLE_NAME); + } + + @Test + public void testGlobalIndexHasGenerateColumn() throws Exception { + /* + CREATE TABLE `test_global_index_with_vgen_col` ( + `name` varchar(512) NOT NULL DEFAULT '', + `pk` varchar(512) NOT NULL, + `adiu` varchar(512) NOT NULL DEFAULT '', + `id` bigint(20) NOT NULL DEFAULT 0, + `name_v` varchar(20) GENERATED ALWAYS AS (substr(`name`,1,5)) VIRTUAL, + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`adiu`, `pk`, `gmt_create`), + KEY `idx_adiu_v_name` (`adiu`) global + ) TTL (gmt_create + INTERVAL 300 SECOND) partition by key(adiu) partitions 8; + */ + String tableName = "test_global_index_with_vgen_col"; + testIndexHasGenerateColumn(tableName); + } + + public void testIndexHasGenerateColumn(String tableName) throws SQLException { + long cur_ts = System.currentTimeMillis(); + Timestamp cur_time = new Timestamp(cur_ts); + cur_time.setNanos(0); + Timestamp expire_time = new Timestamp(cur_ts - 301000); + expire_time.setNanos(0); + try { + Row rowKey1 = row(colVal("adiu", "adiu111"), colVal("pk", "pk1"), + colVal("gmt_create", cur_time)); + Row rowKey2 = row(colVal("adiu", "adiu222"), colVal("pk", "pk2"), + colVal("gmt_create", expire_time)); + Row mutateRow1 = row(colVal("id", 1L), colVal("name", "name1111111")); + Row mutateRow2 = row(colVal("id", 2L), colVal("name", "name2222222")); + + // prepare data + MutationResult res = client.insert(tableName).setRowKey(rowKey1) + .addMutateRow(mutateRow1).execute(); + assertEquals(1, res.getAffectedRows()); + res = client.insert(tableName).setRowKey(rowKey2).addMutateRow(mutateRow2).execute(); + assertEquals(1, res.getAffectedRows()); + // get + // not expired row + Map getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals("adiu111", getRes.get("adiu")); + assertEquals("pk1", getRes.get("pk")); + assertEquals(cur_time, getRes.get("gmt_create")); + assertEquals(1L, getRes.get("id")); + assertEquals("name1111111", getRes.get("name")); + assertEquals("name1", getRes.get("name_v")); + // expired row + getRes = client.get(tableName, rowKey2.getValues(), null); + assertTrue(getRes.isEmpty()); + + // insertup + // not expired: update + res = client.insertOrUpdate(tableName).setRowKey(rowKey1).addMutateRow(mutateRow2) + .execute(); + assertEquals(1, res.getAffectedRows()); + // expired: delete + insert + res = client.insertOrUpdate(tableName).setRowKey(rowKey2).addMutateRow(mutateRow1) + .execute(); + assertEquals(1, res.getAffectedRows()); + // get + getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals("adiu111", getRes.get("adiu")); + assertEquals("pk1", getRes.get("pk")); + assertEquals(cur_time, getRes.get("gmt_create")); + assertEquals(2L, getRes.get("id")); + assertEquals("name2222222", getRes.get("name")); + assertEquals("name2", getRes.get("name_v")); + // expired row + getRes = client.get(tableName, rowKey2.getValues(), null); + assertTrue(getRes.isEmpty()); + + // update + res = client.update(tableName).setRowKey(rowKey1).addMutateRow(mutateRow1).execute(); + assertEquals(1, res.getAffectedRows()); + getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals("adiu111", getRes.get("adiu")); + assertEquals("pk1", getRes.get("pk")); + assertEquals(cur_time, getRes.get("gmt_create")); + assertEquals(1L, getRes.get("id")); + assertEquals("name1111111", getRes.get("name")); + assertEquals("name1", getRes.get("name_v")); + + // replace + res = client.replace(tableName).setRowKey(rowKey1).addMutateRow(mutateRow2).execute(); + assertEquals(2, res.getAffectedRows()); + getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals("adiu111", getRes.get("adiu")); + assertEquals("pk1", getRes.get("pk")); + assertEquals(cur_time, getRes.get("gmt_create")); + assertEquals(2L, getRes.get("id")); + assertEquals("name2222222", getRes.get("name")); + assertEquals("name2", getRes.get("name_v")); + + // increment + Row incrMutateRow = row(colVal("id", 100L)); + res = client.increment(tableName).setRowKey(rowKey1).addMutateRow(incrMutateRow) + .execute(); + assertEquals(1, res.getAffectedRows()); + getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals("adiu111", getRes.get("adiu")); + assertEquals("pk1", getRes.get("pk")); + assertEquals(cur_time, getRes.get("gmt_create")); + assertEquals(102L, getRes.get("id")); + assertEquals("name2222222", getRes.get("name")); + assertEquals("name2", getRes.get("name_v")); + + // append + Row appendMutateRow = row(colVal("name", "ssssss")); + res = client.append(tableName).setRowKey(rowKey1).addMutateRow(appendMutateRow) + .execute(); + assertEquals(1, res.getAffectedRows()); + getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals("adiu111", getRes.get("adiu")); + assertEquals("pk1", getRes.get("pk")); + assertEquals(cur_time, getRes.get("gmt_create")); + assertEquals(102L, getRes.get("id")); + assertEquals("name2222222ssssss", getRes.get("name")); + assertEquals("name2", getRes.get("name_v")); + + // delete + res = client.delete(tableName).setRowKey(rowKey1).execute(); + assertEquals(1, res.getAffectedRows()); + client.delete(tableName).setRowKey(rowKey2).execute(); + assertEquals(1, res.getAffectedRows()); + } catch (Exception ex) { + ex.printStackTrace(); + assertFalse(true); + } finally { + ObTableClientTestUtil.getConnection().createStatement() + .execute("truncate table " + tableName); + } + } + + @Test + public void test_current_ts() throws Exception { + String tableName = "test_current_timestamp"; + try { + long cur_ts = System.currentTimeMillis(); + Timestamp ts_1 = new Timestamp(cur_ts + 1000000); + ts_1.setNanos(0); + Timestamp ts_2 = new Timestamp(cur_ts - 1000000); + ts_2.setNanos(0); + Row rowKey1 = row(colVal("c1", 1)); + Row mutateRow1 = row(colVal("c2", "c2_val"), colVal("c3", ts_1)); + Row mutateRow2 = row(colVal("c2", "c2_val_1"), colVal("c3", ts_2)); + + // insert + MutationResult res = client.insert(tableName).setRowKey(rowKey1) + .addMutateRow(mutateRow1).execute(); + assertEquals(1, res.getAffectedRows()); + // get + Map getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals(1, getRes.get("c1")); + assertEquals("c2_val", getRes.get("c2")); + assertEquals(ts_1, getRes.get("c3")); + + // insertup: update + res = client.insertOrUpdate(tableName).setRowKey(rowKey1).addMutateRow(mutateRow2) + .execute(); + assertEquals(1, res.getAffectedRows()); + // get + getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals(1, getRes.get("c1")); + assertEquals("c2_val_1", getRes.get("c2")); + assertEquals(ts_2, getRes.get("c3")); + + // replace: update + res = client.replace(tableName).setRowKey(rowKey1).addMutateRow(mutateRow1).execute(); + assertEquals(2, res.getAffectedRows()); + // get + getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals(1, getRes.get("c1")); + assertEquals("c2_val", getRes.get("c2")); + assertEquals(ts_1, getRes.get("c3")); + + // update + res = client.update(tableName).setRowKey(rowKey1).addMutateRow(mutateRow2).execute(); + assertEquals(1, res.getAffectedRows()); + // get + getRes = client.get(tableName, rowKey1.getValues(), null); + assertTrue(!getRes.isEmpty()); + assertEquals(1, getRes.get("c1")); + assertEquals("c2_val_1", getRes.get("c2")); + assertEquals(ts_2, getRes.get("c3")); + } catch (Exception ex) { + ex.printStackTrace(); + assertTrue(false); + } finally { + ObTableClientTestUtil.getConnection().createStatement() + .execute("truncate table " + tableName); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java new file mode 100644 index 00000000..d1f1b9f8 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableLsBatchTest.java @@ -0,0 +1,751 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2024 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import com.alipay.oceanbase.rpc.util.TimeUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.*; + +public class ObTableLsBatchTest { + public ObTableClient client; + private static final String TABLE_NAME = "test_mutation"; + + @Before + public void setup() throws Exception { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + } + + /* + CREATE TABLE `test_mutation` ( + `c1` bigint NOT NULL, + `c2` varchar(20) NOT NULL, + `c3` varbinary(1024) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`c1`, `c2`)) partition by range columns (`c1`) ( + PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), + PARTITION p2 VALUES LESS THAN MAXVALUE); + */ + @Test + public void testInsertUp() throws Exception { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + InsertOrUpdate insertOrUpdate = new InsertOrUpdate(); + insertOrUpdate.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insertOrUpdate.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insertOrUpdate); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testGet() throws Exception { + // prepare data + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + InsertOrUpdate insertOrUpdate = client.insertOrUpdate(TABLE_NAME); + insertOrUpdate.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insertOrUpdate.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + MutationResult res = insertOrUpdate.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + } + + try { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + TableQuery query = query().setRowKey( + row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))).select("c1", "c2", "c3", + "c4"); + batchOperation.addOperation(query); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Row row = batchOperationResult.get(j).getOperationRow(); + Assert.assertEquals(values[j][0], row.get("c1")); + Assert.assertEquals(values[j][1], row.get("c2")); + Assert.assertEquals(values[j][2], new String((byte[]) row.get("c3"), "UTF-8")); + Assert.assertEquals(values[j][3], row.get("c4")); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + /* + CREATE TABLE IF NOT EXISTS `test_table_object` ( + `c1` tinyint primary key, + `c2` smallint not null, + `c3` int not null, + `c4` bigint not null, + `c5` varchar(128) not null, + `c6` varbinary(128) not null, + `c7` float not null, + `c8` double not null, + `c9` timestamp(6) not null, + `c10` datetime(6) not null, + `c11` int default null, + `c12` tinytext DEFAULT NULL, + `c13` text DEFAULT NULL, + `c14` mediumtext DEFAULT NULL, + `c15` longtext DEFAULT NULL, + `c16` tinyblob DEFAULT NULL, + `c17` blob DEFAULT NULL, + `c18` mediumblob DEFAULT NULL, + `c19` longblob DEFAULT NULL + ); + */ + @Test + public void testGetAllObjType() throws Exception { + String ALL_OBJ_TYPE_TABLE = "test_table_object"; + + // pre-clean data + client.delete(ALL_OBJ_TYPE_TABLE).addScanRange(ObObj.getMin(), ObObj.getMax()).execute(); + + long timeInMillis = System.currentTimeMillis(); + Timestamp c9Val = new Timestamp(timeInMillis); + Date c10Val = TimeUtils.strToDate("2024-01-30"); + + // prepare data + Object values[][] = { + {(byte)1, (short)1, (int)1, 1L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)2, (short)2, (int)2, 2L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)3, (short)3, (int)3, 3L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)4, (short)4, (int)4, 4L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)5, (short)5, (int)5, 5L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()}, + {(byte)6, (short)6, (int)6, 6L, "c5_val", "c6_val".getBytes(), 100.0f, 200.0d, c9Val, c10Val, null, + "c12_val", "c13_val", "c14_val", "c15_val", "c16_val".getBytes(), "c17_val".getBytes(), + "c18_val".getBytes(), "c19_val".getBytes()} +}; + + int rowCnt = values.length; + + try { + // pre insert data + { + BatchOperation batchOperation = client.batchOperation(ALL_OBJ_TYPE_TABLE); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + InsertOrUpdate insertOrUpdate = new InsertOrUpdate(); + insertOrUpdate.setRowKey(row(colVal("c1", curRow[0]))); + + for (int j = 2; j <= curRow.length; j++) { + insertOrUpdate.addMutateRow(row(colVal("c" + j, curRow[j-1]))); + } + batchOperation.addOperation(insertOrUpdate); + } + BatchOperationResult res = batchOperation.execute(); + for (int k = 0; k < rowCnt; k++) { + Assert.assertEquals(1, res.get(k).getAffectedRows()); + } + } + + // get data with all columns + { + BatchOperation batchOperation = client.batchOperation(ALL_OBJ_TYPE_TABLE); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + TableQuery query = query().setRowKey(row(colVal("c1", curRow[0]))) + .select("c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10", + "c11", "c12", "c13", "c14", "c15", "c16", "c17", "c18", "c19"); + batchOperation.addOperation(query); + } + BatchOperationResult res = batchOperation.execute(); + + for (int j = 0; j < rowCnt; j++) { + Object[] curRow = values[j]; + Row row = res.get(j).getOperationRow(); + for (int k = 0; k < curRow.length; k++) { + Object getValue = row.get("c" + (k+1)); + if (getValue instanceof byte[]) { + Assert.assertTrue(Arrays.equals((byte[]) curRow[k], (byte[]) getValue)); + } else { + Assert.assertEquals(curRow[k], getValue); + } + } + } + } + + // get data with different columns + { + // get columns idx, 1 means get c1 + int columns[][] = { + {3, 8, 14, 11, 18, 5, 12, 6, 19, 10, 15, 7, 1, 4, 9, 13, 2, 16, 17}, + {5, 15, 6, 10, 2, 12, 7, 17, 18, 11, 3, 8, 13, 1, 16, 4, 9, 14}, + {13, 10, 5, 15, 1, 8, 11, 9, 7, 16, 4, 17, 6, 2, 12, 3, 14}, + {14, 6, 9, 2, 5, 3, 15, 10, 12, 8, 11, 7, 1, 16, 4, 13}, + {11, 3, 7, 12, 8, 10, 4, 6, 15, 2, 5, 1, 14, 9, 13}, + {8, 12, 5, 14, 3, 7, 1, 11, 6, 9, 10, 2, 4, 13}, + {10, 4, 13, 1, 8, 7, 5, 11, 6, 3, 12, 2, 9}, + {9, 3, 10, 5, 7, 1, 8, 6, 11, 4, 12, 2}, + {7, 3, 11, 5, 6, 2, 9, 1, 8, 4, 10}, + {5, 2, 7, 9, 1, 8, 6, 3, 10, 4}, + {5, 3, 1, 8, 6, 4, 7, 2, 9}, + {3, 2, 5, 8, 7, 1, 6, 4}, + {4, 6, 7, 1, 3, 5, 2}, + {2, 6, 3, 1, 4, 5}, + }; + + BatchOperation batchOperation = client.batchOperation(ALL_OBJ_TYPE_TABLE); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + TableQuery query = query().setRowKey(row(colVal("c1", curRow[0]))); + List selectColumns = new ArrayList<>(); + for (int j = 0; j < columns[i].length; j++) { + selectColumns.add("c" + columns[i][j]); + } + query.select(selectColumns.toArray(new String[0])); + batchOperation.addOperation(query); + } + BatchOperationResult res = batchOperation.execute(); + + for (int j = 0; j < rowCnt; j++) { + int curColumns[] = columns[j]; + Object[] curRow = values[j]; + Row row = res.get(j).getOperationRow(); + for (int k = 0; k < curColumns.length; k++) { + Object curValue = curRow[curColumns[k]-1]; + String curSelectColumn = "c" + curColumns[k]; + Object getValue = row.get(curSelectColumn); + if (getValue instanceof byte[]) { + Assert.assertTrue(Arrays.equals((byte[]) curValue, (byte[]) getValue)); + } else { + Assert.assertEquals(curValue, getValue); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally {} + } + + @Test + public void testBatchInsert() throws Exception { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(1, batchOperationResult.size()); + Assert.assertEquals(rowCnt, batchOperationResult.get(0).getAffectedRows()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchAppend() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // append + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Append append = new Append(); + append.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + append.addMutateRow(row(colVal("c3", curRow[2]))); + batchOperation.addOperation(append); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchDel() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // del + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Delete delete = new Delete(); + delete.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + batchOperation.addOperation(delete); + } + + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(1, batchOperationResult.size()); + // todo: the multi-delete's affected_rows is inaccuracy cuz of server's bug + // Assert.assertEquals(rowCnt, batchOperationResult.get(0).getAffectedRows()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } + } + + @Test + public void testBatchIncrement() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // increment + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Increment inc = new Increment(); + inc.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + inc.addMutateRow(row(colVal("c4", curRow[3]))); + batchOperation.addOperation(inc); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchReplace() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // replace + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Replace replace = new Replace(); + replace.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + replace.addMutateRow(row(colVal("c3", curRow[1]), colVal("c4", curRow[3]))); + batchOperation.addOperation(replace); + } + + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(1, batchOperationResult.size()); + Assert.assertEquals(rowCnt * 2, batchOperationResult.get(0).getAffectedRows()); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testBatchUpdate() throws Exception { + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 400L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 1000L, "c2_val", "c3_val", 100L }, + { 1001L, "c2_val", "c3_val", 100L }, { 1002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + try { + // insert + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + insert.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(insert); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + // update + { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Update update = new Update(); + update.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + update.addMutateRow(row(colVal("c3", curRow[1]), colVal("c4", curRow[3]))); + batchOperation.addOperation(update); + } + + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(rowCnt, batchOperationResult.size()); + for (int j = 0; j < rowCnt; j++) { + Assert.assertEquals(1, batchOperationResult.get(j).getAffectedRows()); + } + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } + + @Test + public void testHybridBatch() throws Exception { + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + + try { + // 1. insert {1L, "c2_val", "c3_val", 100L} + { + Insert insert = new Insert(); + insert.setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_val"))); + insert.addMutateRow(row(colVal("c3", "c3_val"), colVal("c4", 100L))); + batchOperation.addOperation(insert); + } + + // 2. append {1L, "c2_val", "_val"} + { + Append append = new Append(); + append.setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_val"))); + append.addMutateRow(row(colVal("c3", "_val"))); + batchOperation.addOperation(append); + } + + // 3. get {1L, "c2_val"} + { + TableQuery query1 = query() + .setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_val"))).select("c1", "c2", + "c3", "c4"); + batchOperation.addOperation(query1); + } + + // 4. delete {1L, "c2_val"} + { + Delete delete = new Delete(); + delete.setRowKey(row(colVal("c1", 1L), colVal("c2", "c2_val"))); + batchOperation.addOperation(delete); + + } + + // 5. increment {1001L, "c2_val", 100L} + { + Increment inc = new Increment(); + inc.setRowKey(row(colVal("c1", 1001L), colVal("c2", "c2_val"))); + inc.addMutateRow(row(colVal("c4", 100L))); + batchOperation.addOperation(inc); + } + + // 6. insertUp {1001L, "c2_val", "c3_val", 100L} + { + InsertOrUpdate insertOrUpdate = new InsertOrUpdate(); + insertOrUpdate.setRowKey(row(colVal("c1", 1001L), colVal("c2", "c2_val"))); + insertOrUpdate.addMutateRow(row(colVal("c3", "c3_val"), colVal("c4", 200L))); + batchOperation.addOperation(insertOrUpdate); + } + + // 7. replace {1001L, "c2_val", "c3_val", 300L} + { + Replace replace = new Replace(); + replace.setRowKey(row(colVal("c1", 1001L), colVal("c2", "c2_val"))); + replace.addMutateRow(row(colVal("c3", "c3_val"), colVal("c4", 300L))); + batchOperation.addOperation(replace); + } + + // 8. update {1001L, "c2_val", "c3_val_val", 400L} + { + Update update = new Update(); + update.setRowKey(row(colVal("c1", 1001L), colVal("c2", "c2_val"))); + update.addMutateRow(row(colVal("c3", "c3_val_val"), colVal("c4", 400L))); + batchOperation.addOperation(update); + } + + // 9. get {1001L, "c2_val"} + { + TableQuery query2 = query().setRowKey( + row(colVal("c1", 1001L), colVal("c2", "c2_val"))) + .select("c1", "c2", "c3", "c4"); + batchOperation.addOperation(query2); + } + + BatchOperationResult result = batchOperation.execute(); + + int affectRows[] = { 1, 1, 0, 1, 1, 1, 2, 1, 0 }; + for (int i = 0; i < affectRows.length; i++) { + Assert.assertEquals(affectRows[i], result.get(i).getAffectedRows()); + } + + Object getExpRows[][] = { { 2 /*res idx*/, 1L, "c2_val", "c3_val_val", 100L }, + { 8 /*res idx*/, 1001L, "c2_val", "c3_val_val", 400L } }; + + for (int j = 0; j < getExpRows.length; j++) { + Object curRow[] = getExpRows[j]; + Row row = result.get((int) curRow[0]).getOperationRow(); + Assert.assertEquals(curRow[1], row.get("c1")); + Assert.assertEquals(curRow[2], row.get("c2")); + Assert.assertEquals(curRow[3], new String((byte[]) row.get("c3"), "UTF-8")); + Assert.assertEquals(curRow[4], row.get("c4")); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + Object rowkeys[][] = { { 1L, "c2_val" }, { 1001L, "c2_val" }, }; + for (int i = 0; i < rowkeys.length; i++) { + Object curRow[] = rowkeys[i]; + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + delete.execute(); + } + } + } + + @Test + public void testPut() throws Exception { + // put operation should set binlog_row_image minimal + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("SET GLOBAL binlog_row_image= 'minimal'"); + + BatchOperation batchOperation = client.batchOperation(TABLE_NAME); + Object values[][] = { { 1L, "c2_val", "c3_val", 100L }, { 200L, "c2_val", "c3_val", 100L }, + { 401L, "c2_val", "c3_val", 100L }, { 2000L, "c2_val", "c3_val", 100L }, + { 100001L, "c2_val", "c3_val", 100L }, { 10000002L, "c2_val", "c3_val", 100L }, }; + int rowCnt = values.length; + + try { + for (int i = 0; i < rowCnt; i++) { + Object[] curRow = values[i]; + Put put = new Put(); + put.setRowKey(row(colVal("c1", curRow[0]), colVal("c2", curRow[1]))); + put.addMutateRow(row(colVal("c3", curRow[2]), colVal("c4", curRow[3]))); + batchOperation.addOperation(put); + } + batchOperation.setReturnOneResult(true); + BatchOperationResult batchOperationResult = batchOperation.execute(); + Assert.assertEquals(1, batchOperationResult.size()); + Assert.assertEquals(rowCnt, batchOperationResult.get(0).getAffectedRows()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + statement.execute("SET GLOBAL binlog_row_image= 'FULL'"); + for (int j = 0; j < rowCnt; j++) { + Delete delete = client.delete(TABLE_NAME); + delete.setRowKey(row(colVal("c1", values[j][0]), colVal("c2", values[j][1]))); + MutationResult res = delete.execute(); + Assert.assertEquals(1, res.getAffectedRows()); + } + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableModeTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableModeTest.java new file mode 100644 index 00000000..cfe9642e --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableModeTest.java @@ -0,0 +1,316 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.filter.ObCompareOp; +import com.alipay.oceanbase.rpc.filter.ObTableFilterList; +import com.alipay.oceanbase.rpc.filter.ObTableValueFilter; +import com.alipay.oceanbase.rpc.mutation.Append; +import com.alipay.oceanbase.rpc.mutation.BatchOperation; +import com.alipay.oceanbase.rpc.mutation.Insert; +import com.alipay.oceanbase.rpc.mutation.InsertOrUpdate; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateRequest; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Random; + +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.andList; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static com.alipay.oceanbase.rpc.util.ObTableClientTestUtil.*; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class ObTableModeTest { + ObTableClient client; + public static String tableName = "test_varchar_table"; + public static String mysqlCompatMode = "mysql"; + public static String oracleCompatMode = "oracle"; + + public static String allKVMode = "ALL"; + public static String tableKVMode = "TABLEAPI"; + public static String hbaseKVMode = "HBASE"; + public static String RedisKVMode = "REDIS"; + public static String noneKVMode = "NONE"; + + public static String tpUnit = "tpUnit"; + public static String tpPool = "tpPool"; + public static String tpTenant = "tpTenant"; + + public static String hbaseUnit = "hbaseUnit"; + public static String hbasePool = "hbasePool"; + public static String hbaseTenant = "hbaseTenant"; + + public static String tableUnit = "tableUnit"; + public static String tablePool = "tablePool"; + public static String tableTenant = "tableTenant"; + + public static String redisUnit = "redisUnit"; + public static String redisPool = "redisPool"; + public static String redisTenant = "redisTenant"; + + @Before + public void setup() throws Exception { + } + + public static String extractUserName(String input) { + int atSymbolIndex = input.indexOf("@"); + if (atSymbolIndex != -1) { + return input.substring(0, atSymbolIndex); + } else { + return ""; + } + } + + public static String extractClusterName(String input) { + int hashSymbolIndex = input.lastIndexOf("#"); + if (hashSymbolIndex != -1) { + return input.substring(hashSymbolIndex + 1); + } else { + return ""; + } + } + + public void createTable(String userName, String tenantName) throws Exception { + String user = userName + "@" + tenantName; + String url = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + "test" + "?" + + "rewriteBatchedStatements=TRUE&" + + "allowMultiQueries=TRUE&" + + "useLocalSessionState=TRUE&" + + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + + "connectTimeout=60000"; + Connection conn = DriverManager.getConnection(url, user, PASSWORD); + Statement statement = conn.createStatement(); + statement.execute("CREATE TABLE IF NOT EXISTS `test_varchar_table` (" + + " `c1` varchar(20) NOT NULL," + + " `c2` varchar(20) DEFAULT NULL," + + " PRIMARY KEY (`c1`)" + + " );"); + } + + public void createResourceUnit(String unitName) throws Exception { + Connection conn = ObTableClientTestUtil.getSysConnection(); + Statement statement = conn.createStatement(); + statement.execute("create resource unit " + unitName + " max_cpu 1, memory_size '1G';"); + } + + public void createResourcePool(String unitName, String poolName) throws Exception { + createResourceUnit(unitName); + Connection conn = ObTableClientTestUtil.getSysConnection(); + Statement statement = conn.createStatement(); + statement.execute("create resource pool " + poolName + " unit = '" + unitName + "', unit_num = 1;"); + } + + public void createTenant(String unitName, String poolName, String tenantName, String compatMode, String kvMode) throws Exception { + createResourcePool(unitName, poolName); + Connection conn = ObTableClientTestUtil.getSysConnection(); + Statement statement = conn.createStatement(); + statement.execute("create tenant " + tenantName + " replica_num = 1, resource_pool_list=('"+ poolName +"') " + + "set ob_tcp_invited_nodes='%', ob_compatibility_mode='" + compatMode + "', ob_kv_mode='" + kvMode + "';"); + } + + public void dropResourceUnit(String unitName) throws Exception { + Connection conn = ObTableClientTestUtil.getSysConnection(); + Statement statement = conn.createStatement(); + statement.execute("drop resource unit if exists " + unitName + ";"); + } + + public void dropResourcePool(String poolName) throws Exception { + Connection conn = ObTableClientTestUtil.getSysConnection(); + Statement statement = conn.createStatement(); + statement.execute("drop resource pool if exists " + poolName + ";"); + } + + public void dropTenant(String tenantName) throws Exception { + Connection conn = ObTableClientTestUtil.getSysConnection(); + Statement statement = conn.createStatement(); + statement.execute("drop tenant if exists " + tenantName + " force;"); + } + + /** + CREATE TABLE IF NOT EXISTS `test_varchar_table` ( + `c1` varchar(20) NOT NULL, + `c2` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + **/ + @Test + public void testTpTenant() throws Exception { + try { + createTenant(tpUnit, tpPool, tpTenant, mysqlCompatMode, noneKVMode); + + String tenantName = tpTenant; + String userName = extractUserName(FULL_USER_NAME); + String clusterName = extractClusterName(FULL_USER_NAME); + String fullName = userName + "@" + tenantName + "#" + clusterName; + + createTable(userName, tenantName); + + ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.setFullUserName(fullName); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { "c1" }); + + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.insert(tableName).setRowKey(colVal("c1", "a")) + .addMutateColVal(colVal("c2", "a")) + .execute(); + } + ); + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][As the ob_kv_mode variable has been set to 'NONE', your current interfaces not supported]")); + } finally { + dropTenant(tpTenant); + dropResourcePool(tpPool); + dropResourceUnit(tpUnit); + } + } + + /** + CREATE TABLE IF NOT EXISTS `test_varchar_table` ( + `c1` varchar(20) NOT NULL, + `c2` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + **/ + @Test + public void testHbaseTenant() throws Exception { + try { + String tenantName = hbaseTenant; + createTenant(hbaseUnit, hbasePool, tenantName, mysqlCompatMode, hbaseKVMode); + + String userName = extractUserName(FULL_USER_NAME); + String clusterName = extractClusterName(FULL_USER_NAME); + String fullName = userName + "@" + tenantName + "#" + clusterName; + + createTable(userName, tenantName); + + ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.setFullUserName(fullName); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { "c1" }); + + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.insert(tableName).setRowKey(colVal("c1", "a")) + .addMutateColVal(colVal("c2", "a")) + .execute(); + } + ); + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][As the ob_kv_mode variable has been set to 'HBASE', your current interfaces not supported]")); + } finally { + dropTenant(hbaseTenant); + dropResourcePool(hbasePool); + dropResourceUnit(hbaseUnit); + } + } + + /** + CREATE TABLE IF NOT EXISTS `test_varchar_table` ( + `c1` varchar(20) NOT NULL, + `c2` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + **/ + @Test + public void testTableTenant() throws Exception { + try { + String tenantName = tableTenant; + createTenant(tableUnit, tablePool, tenantName, mysqlCompatMode, tableKVMode); + + String userName = extractUserName(FULL_USER_NAME); + String clusterName = extractClusterName(FULL_USER_NAME); + String fullName = userName + "@" + tenantName + "#" + clusterName; + + createTable(userName, tenantName); + + ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.setFullUserName(fullName); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { "c1" }); + + client.insert(tableName).setRowKey(colVal("c1", "a")) + .addMutateColVal(colVal("c2", "a")) + .execute(); + } finally { + dropTenant(tableTenant); + dropResourcePool(tablePool); + dropResourceUnit(tableUnit); + } + } + + /** + CREATE TABLE IF NOT EXISTS `test_varchar_table` ( + `c1` varchar(20) NOT NULL, + `c2` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + **/ + @Test + public void testRedisTenant() throws Exception { + try { + String tenantName = redisTenant; + createTenant(redisUnit, redisPool, tenantName, mysqlCompatMode, RedisKVMode); + + String userName = extractUserName(FULL_USER_NAME); + String clusterName = extractClusterName(FULL_USER_NAME); + String fullName = userName + "@" + tenantName + "#" + clusterName; + + createTable(userName, tenantName); + + ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.setFullUserName(fullName); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { "c1" }); + + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.insert(tableName).setRowKey(colVal("c1", "a")) + .addMutateColVal(colVal("c2", "a")) + .execute(); + } + ); + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][As the ob_kv_mode variable has been set to 'REDIS', your current interfaces not supported]")); + } finally { + dropTenant(redisTenant); + dropResourcePool(redisPool); + dropResourceUnit(redisUnit); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableP99Test.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableP99Test.java new file mode 100644 index 00000000..07f75d97 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableP99Test.java @@ -0,0 +1,385 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.protocol.payload.ObPayload; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateRequest; +import com.alipay.oceanbase.rpc.table.api.TableBatchOps; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.List; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static org.junit.Assert.*; + +/** + CREATE TABLE IF NOT EXISTS `test_p99` ( + `c1` bigint(20) NOT NULL, + `c2` bigint(20) DEFAULT NULL, + `c3` varchar(20) DEFAULT "hello", + PRIMARY KEY (`c1`) + ); + **/ +public class ObTableP99Test { + ObTableClient client; + private static Connection conn = null; + public static String tableName = "test_p99"; + public static String insertSqlType = "TABLEAPI INSERT"; + public static String selectSqlType = "TABLEAPI SELECT"; + public static String deleteSqlType = "TABLEAPI DELETE"; + public static String updateSqlType = "TABLEAPI UPDATE"; + public static String replaceSqlType = "TABLEAPI REPLACE"; + public static String queryAndMutateSqlType = "TABLEAPI QUERY AND MUTATE"; + public static String otherSqlType = "TABLEAPI OTHER"; + + @Before + public void setup() throws Exception { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { "c1" }); + conn = ObTableClientTestUtil.getConnection(); + } + + private static long getResultCount(String sqlType) throws Exception { + PreparedStatement ps = conn.prepareStatement("select * from oceanbase.gv$ob_query_response_time_histogram " + + "where sql_type=" + "\"" +sqlType +"\""); + ResultSet rs = ps.executeQuery(); + long totalCnt = 0L; + while (rs.next()) { + totalCnt += rs.getLong("count"); + } + ps.close(); + return totalCnt; + } + + private static void flushHistogram() throws Exception { + Statement statement = conn.createStatement(); + statement.execute("alter system set query_response_time_flush = true;"); + } + + @Test + public void testInsert() throws Exception { + try { + // single insert + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + assertEquals(1, getResultCount(insertSqlType)); + client.insert(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + assertEquals(2, getResultCount(insertSqlType)); + + // single insertOrUpdate + flushHistogram(); + Thread.sleep(100); + client.insertOrUpdate(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + assertEquals(1, getResultCount(insertSqlType)); + client.insertOrUpdate(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + assertEquals(2, getResultCount(insertSqlType)); + + // multi insert + flushHistogram(); + Thread.sleep(100); + BatchOperation batch1 = client.batchOperation(tableName); + Insert ins_0 = client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)); + batch1.addOperation(ins_0).execute(); + assertEquals(1, getResultCount(insertSqlType)); + batch1.addOperation(ins_0).execute(); + assertEquals(2, getResultCount(insertSqlType)); + + // multi insertOrUpdate + flushHistogram(); + Thread.sleep(100); + BatchOperation batch2 = client.batchOperation(tableName); + InsertOrUpdate insUp_0 = client.insertOrUpdate(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)); + batch2.addOperation(insUp_0).execute(); + assertEquals(1, getResultCount(insertSqlType)); + batch2.addOperation(insUp_0).execute(); + assertEquals(2, getResultCount(insertSqlType)); + + } finally { + client.delete(tableName).setRowKey(colVal("c1", 1L)).execute(); + client.delete(tableName).setRowKey(colVal("c1", 2L)).execute(); + flushHistogram(); + } + } + + @Test + public void testSelect() throws Exception { + try { + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + assertEquals(1, getResultCount(insertSqlType)); + + // single get + flushHistogram(); + Thread.sleep(100); + client.get(tableName, 1L, new String[]{"c1"}); + assertEquals(1, getResultCount(selectSqlType)); + client.get(tableName, 1L, new String[]{"c1"}); + assertEquals(2, getResultCount(selectSqlType)); + + // multi get + flushHistogram(); + Thread.sleep(100); + TableBatchOps batchOps = client.batch(tableName); + batchOps.get(1L, new String[]{"c1"}); + batchOps.execute(); + assertEquals(1, getResultCount(selectSqlType)); + batchOps.execute(); + assertEquals(2, getResultCount(selectSqlType)); + + // sync query + flushHistogram(); + Thread.sleep(100); + TableQuery tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L,}, new Object[] { 1L,}); + tableQuery.select("c1"); + tableQuery.execute(); + assertEquals(1, getResultCount(selectSqlType)); + tableQuery.execute(); + assertEquals(2, getResultCount(selectSqlType)); + + // async query + flushHistogram(); + Thread.sleep(100); + tableQuery.asyncExecute(); + assertEquals(1, getResultCount(selectSqlType)); + tableQuery.asyncExecute(); + assertEquals(2, getResultCount(selectSqlType)); + } finally { + client.delete(tableName).setRowKey(colVal("c1", 1L)).execute(); + flushHistogram(); + } + } + + @Test + public void testDelete() throws Exception { + try { + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + client.insert(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + assertEquals(2, getResultCount(insertSqlType)); + + // single delete + flushHistogram(); + Thread.sleep(100); + client.delete(tableName).setRowKey(colVal("c1", 1L)).execute(); + assertEquals(1, getResultCount(deleteSqlType)); + client.delete(tableName).setRowKey(colVal("c1", 2L)).execute(); + assertEquals(2, getResultCount(deleteSqlType)); + + // multi delete + flushHistogram(); + Thread.sleep(100); + BatchOperation batch = client.batchOperation(tableName); + Delete del_0 = client.delete(tableName).setRowKey(colVal("c1", 1L)); + batch.addOperation(del_0).execute(); + assertEquals(1, getResultCount(deleteSqlType)); + batch.addOperation(del_0).execute(); + assertEquals(2, getResultCount(deleteSqlType)); + } finally { + client.delete(tableName).setRowKey(colVal("c1", 1L)).execute(); + client.delete(tableName).setRowKey(colVal("c1", 2L)).execute(); + flushHistogram(); + } + } + + @Test + public void testUpdate() throws Exception { + try { + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + client.insert(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + assertEquals(2, getResultCount(insertSqlType)); + + // single update + flushHistogram(); + Thread.sleep(100); + client.update(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 2L)) + .execute(); + assertEquals(1, getResultCount(updateSqlType)); + client.update(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", 2L)) + .execute(); + assertEquals(2, getResultCount(updateSqlType)); + + // single increment + flushHistogram(); + Thread.sleep(100); + client.increment(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 2L)) + .execute(); + assertEquals(1, getResultCount(updateSqlType)); + client.increment(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", 2L)) + .execute(); + assertEquals(2, getResultCount(updateSqlType)); + + // single append + flushHistogram(); + Thread.sleep(100); + client.append(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c3", "world")) + .execute(); + assertEquals(1, getResultCount(updateSqlType)); + client.append(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c3", "world")) + .execute(); + assertEquals(2, getResultCount(updateSqlType)); + + // multi update + flushHistogram(); + Thread.sleep(100); + BatchOperation batch1 = client.batchOperation(tableName); + Update ins_0 = client.update(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)); + batch1.addOperation(ins_0).execute(); + assertEquals(1, getResultCount(updateSqlType)); + batch1.addOperation(ins_0).execute(); + assertEquals(2, getResultCount(updateSqlType)); + + // multi increment + flushHistogram(); + Thread.sleep(100); + BatchOperation batch2 = client.batchOperation(tableName); + Increment inc_0 = client.increment(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)); + batch2.addOperation(inc_0).execute(); + assertEquals(1, getResultCount(updateSqlType)); + batch2.addOperation(inc_0).execute(); + assertEquals(2, getResultCount(updateSqlType)); + + // multi append + flushHistogram(); + Thread.sleep(100); + BatchOperation batch3 = client.batchOperation(tableName); + Append app_0 = client.append(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c3", "llo")); + batch3.addOperation(app_0).execute(); + assertEquals(1, getResultCount(updateSqlType)); + batch3.addOperation(app_0).execute(); + assertEquals(2, getResultCount(updateSqlType)); + } finally { + client.delete(tableName).setRowKey(colVal("c1", 1L)).execute(); + client.delete(tableName).setRowKey(colVal("c1", 2L)).execute(); + flushHistogram(); + } + } + + @Test + public void testReplace() throws Exception { + try { + client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + client.insert(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", 1L)) + .execute(); + assertEquals(2, getResultCount(insertSqlType)); + flushHistogram(); + Thread.sleep(100); + client.replace(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 2L)) + .execute(); + assertEquals(1, getResultCount(replaceSqlType)); + client.replace(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", 2L)) + .execute(); + assertEquals(2, getResultCount(replaceSqlType)); + + // multi replace + flushHistogram(); + Thread.sleep(100); + BatchOperation batch = client.batchOperation(tableName); + Replace rep_0 = client.replace(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)); + batch.addOperation(rep_0).execute(); + assertEquals(1, getResultCount(replaceSqlType)); + batch.addOperation(rep_0).execute(); + assertEquals(2, getResultCount(replaceSqlType)); + } finally { + client.delete(tableName).setRowKey(colVal("c1", 1L)).execute(); + client.delete(tableName).setRowKey(colVal("c1", 2L)).execute(); + flushHistogram(); + } + } + + @Test + public void testQueryAndMutate() throws Exception { + try { + flushHistogram(); + Thread.sleep(100); + TableQuery tableQuery = client.query(tableName); + tableQuery.addScanRange(new Object[] { 0L,}, new Object[] { 1L,}); + tableQuery.select("c1"); + ObTableQueryAndMutateRequest request_0 = client.obTableQueryAndAppend(tableQuery, + new String[] { "c3" }, new Object[] {"_append0" }, true); + client.execute(request_0); + assertEquals(1, getResultCount(queryAndMutateSqlType)); + } finally { + client.delete(tableName).setRowKey(colVal("c1", 1L)).execute(); + flushHistogram(); + } + } + + @Test + public void testOther() throws Exception { + try { + flushHistogram(); + Thread.sleep(100); + BatchOperation batch = client.batchOperation(tableName); + Insert ins_0 = client.insert(tableName).setRowKey(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 1L)); + Replace rep_0 = client.replace(tableName).setRowKey(colVal("c1", 2L)) + .addMutateColVal(colVal("c2", 1L)); + batch.addOperation(ins_0, rep_0).execute(); + assertEquals(1, getResultCount(otherSqlType)); + batch.addOperation(ins_0, rep_0).execute(); + assertEquals(2, getResultCount(otherSqlType)); + } finally { + client.delete(tableName).setRowKey(colVal("c1", 1L)).execute(); + client.delete(tableName).setRowKey(colVal("c1", 2L)).execute(); + flushHistogram(); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java new file mode 100644 index 00000000..653046ef --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTablePutTest.java @@ -0,0 +1,156 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.Statement; +import java.sql.Timestamp; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +/** + CREATE TABLE IF NOT EXISTS `test_put` ( + `id` varchar(20) NOT NULL, + `c1` bigint DEFAULT NULL, + `c2` bigint DEFAULT NULL, + `c3` varchar(32) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + `expired_ts` timestamp(6) NOT NULL, + PRIMARY KEY(`id`)) TTL(expired_ts + INTERVAL 1 SECOND) PARTITION BY KEY(`id`) PARTITIONS 32; + **/ +public class ObTablePutTest { + ObTableClient client; + public static String tableName = "test_put"; + + @Before + public void setup() throws Exception { + setMinimalImage(); + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { "id" }); + } + + private static void setMinimalImage() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("SET GLOBAL binlog_row_image=MINIMAL"); + } + + private static void setFullImage() throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("SET GLOBAL binlog_row_image=Full"); + } + + @Test + public void testPut1() throws Exception { + try { + Timestamp curTs = new Timestamp(System.currentTimeMillis()); + curTs.setNanos(0); + client.put(tableName).setRowKey(colVal("id", "id0")) + .addMutateColVal(colVal("expired_ts", curTs)).execute(); + client.put(tableName).setRowKey(colVal("id", "id0")).addMutateColVal(colVal("c1", 1L)) + .addMutateColVal(colVal("expired_ts", curTs)).execute(); + client.put(tableName).setRowKey(colVal("id", "id0")).addMutateColVal(colVal("c1", 1L)) + .addMutateColVal(colVal("c2", 2L)).addMutateColVal(colVal("expired_ts", curTs)) + .execute(); + client.put(tableName).setRowKey(colVal("id", "id0")) + .addMutateColVal(colVal("c3", "c3")).addMutateColVal(colVal("expired_ts", curTs)) + .execute(); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + client.delete(tableName).setRowKey(colVal("id", "id0")).execute(); + } + } + + @Test + public void testPut2() { + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + client.put(tableName).setRowKey(colVal("id", "id0")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4227][OB_ERR_NO_DEFAULT_FOR_FIELD][Field 'expired_ts' doesn't have a default value]")); + } + + @Test + public void testPut3() throws Exception { + setFullImage(); + ObTableClient newClient = ObTableClientTestUtil.newTestClient(); + newClient.init(); + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + newClient.put(tableName).setRowKey(colVal("id", "id0")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][binlog_row_image is full use put not supported]")); + } + + @Test + public void testPut4() throws Exception { + setMinimalImage(); + ObTableClient newClient = ObTableClientTestUtil.newTestClient(); + newClient.init(); + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + newClient.put("test_put_with_local_index").setRowKey(colVal("id", "id0")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][table with index use put not supported]")); + } + + @Test + public void testPut5() throws Exception { + setMinimalImage(); + ObTableClient newClient = ObTableClientTestUtil.newTestClient(); + newClient.init(); + ObTableException thrown = assertThrows( + ObTableException.class, + () -> { + newClient.put("test_put_with_global_index").setRowKey(colVal("id", "id0")) + .execute(); + } + ); + + System.out.println(thrown.getMessage()); + assertTrue(thrown.getMessage().contains("[-4007][OB_NOT_SUPPORTED][table with index use put not supported]")); + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableSQLAuditTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableSQLAuditTest.java new file mode 100644 index 00000000..f9d033f3 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableSQLAuditTest.java @@ -0,0 +1,307 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.row; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +// NOTE: this test may not pass when sql audit eliminate frequently, increase your +// observer's memory or sql_audit_memory_limit to alleviate the problem +public class ObTableSQLAuditTest { + class AuditRows { + int affected_rows = -1; + int return_rows = -1; + } + + public ObTableClient client; + + @Before + public void setup() throws Exception { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + } + + // get affected_rows and return rows from sql_audit + AuditRows fetch_audit_rows() throws Exception { + AuditRows auditRows = new AuditRows(); + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement + .execute("select affected_rows, return_rows from oceanbase.gv$sql_audit where query_sql like 'table api%' order by request_time desc limit 1"); + ResultSet resultSet = statement.getResultSet(); + int rowCnt = 0; + while (resultSet.next()) { + auditRows.affected_rows = resultSet.getInt(1); + auditRows.return_rows = resultSet.getInt(2); + rowCnt++; + } + assertEquals(1, rowCnt); + assertEquals(false, resultSet.next()); + return auditRows; + } + + @Test + public void testQuery() throws Exception { + // todo: only support in 3.x currently + if (ObTableClientTestUtil.isOBVersionGreaterEqualThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + String tableName = "test_varchar_table"; + String startKey = "k1"; + String endKey = "k2"; + try { + TableQuery query = client.query(tableName).addScanRange(startKey, endKey); + QueryResultSet res = query.execute(); + Assert.assertEquals(0, res.cacheSize()); + AuditRows auditRows = fetch_audit_rows(); + assertEquals(0, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + MutationResult mutateRes = client.insertOrUpdate(tableName) + .setRowKey(colVal("c1", startKey)).addMutateColVal(colVal("c2", "v1")).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + mutateRes = client.insertOrUpdate(tableName).setRowKey(colVal("c1", endKey)) + .addMutateColVal(colVal("c2", "v1")).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + + query = client.query(tableName).addScanRange(startKey, endKey); + res = query.execute(); + Assert.assertEquals(2, res.cacheSize()); + auditRows = fetch_audit_rows(); + assertEquals(0, auditRows.affected_rows); + assertEquals(2, auditRows.return_rows); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + client.delete(tableName).setRowKey(colVal("c1", startKey)).execute(); + client.delete(tableName).setRowKey(colVal("c1", endKey)).execute(); + } + + } + + @Test + public void test_single_operation() throws Exception { + // todo: only support in 3.x currently + if (ObTableClientTestUtil.isOBVersionGreaterEqualThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + String tableName = "test_varchar_table"; + ColumnValue rowKey = colVal("c1", "k1"); + try { + // 1. get with empty result + Map getRes = client.get(tableName, "not_exist_key", null); + Assert.assertTrue(getRes.isEmpty()); + AuditRows auditRows = fetch_audit_rows(); + assertEquals(0, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + // 2. insert new record + MutationResult mutateRes = client.insert(tableName).setRowKey(rowKey) + .addMutateColVal(colVal("c2", "v1")).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + auditRows = fetch_audit_rows(); + assertEquals(1, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + // 3. insertOrUpdate + mutateRes = client.insertOrUpdate(tableName).setRowKey(rowKey) + .addMutateColVal(colVal("c2", "v2")).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + auditRows = fetch_audit_rows(); + assertEquals(1, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + // 4. update + mutateRes = client.update(tableName).setRowKey(rowKey) + .addMutateColVal(colVal("c2", "v3")).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + auditRows = fetch_audit_rows(); + assertEquals(1, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + // 5. replace + mutateRes = client.replace(tableName).setRowKey(rowKey) + .addMutateColVal(colVal("c2", "v4")).execute(); + assertEquals(2, mutateRes.getAffectedRows()); + auditRows = fetch_audit_rows(); + assertEquals(2, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + // 6. append + mutateRes = client.append(tableName).setRowKey(rowKey) + .addMutateColVal(colVal("c2", "v4")).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + auditRows = fetch_audit_rows(); + assertEquals(1, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + // 7. delete + mutateRes = client.delete(tableName).setRowKey(rowKey).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + auditRows = fetch_audit_rows(); + assertEquals(1, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + // 8. replace + mutateRes = client.replace(tableName).setRowKey(rowKey) + .addMutateColVal(colVal("c2", "v4")).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + auditRows = fetch_audit_rows(); + assertEquals(1, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + client.delete(tableName).setRowKey(rowKey).execute(); + } + + } + + @Test + public void testBatchOperation() throws Exception { + // todo: only support in 3.x currently + if (ObTableClientTestUtil.isOBVersionGreaterEqualThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + String tableName = "test_varchar_table"; + ColumnValue insUpKey = colVal("c1", "k1"); + ColumnValue deleteKey = colVal("c1", "k2"); + ColumnValue updateKey = colVal("c1", "k3"); + ColumnValue appendKey = colVal("c1", "k4"); + ColumnValue getKey = colVal("c2", "k5"); + ColumnValue updateVal = colVal("c2", "v1"); + try { + BatchOperation batchOperation = client.batchOperation(tableName); + InsertOrUpdate insertUp = client.insertOrUpdate(tableName).setRowKey(insUpKey) + .addMutateColVal(updateVal); + Delete delete = client.delete(tableName).setRowKey(deleteKey); + Update update = client.update(tableName).setRowKey(updateKey) + .addMutateColVal(updateVal); + Append append = client.append(tableName).setRowKey(appendKey) + .addMutateColVal(updateVal); + ; + TableQuery query = client.query(tableName).setRowKey(row(getKey)); + batchOperation.addOperation(insertUp, delete, update, append).addOperation(query); + batchOperation.execute(); + + AuditRows auditRows = fetch_audit_rows(); + assertEquals(2, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + client.insertOrUpdate(tableName).setRowKey(getKey).addMutateColVal(updateVal).execute(); + batchOperation.execute(); + + auditRows = fetch_audit_rows(); + assertEquals(2, auditRows.affected_rows); + assertEquals(1, auditRows.return_rows); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + client.delete(tableName).setRowKey(insUpKey).execute(); + client.delete(tableName).setRowKey(deleteKey).execute(); + client.delete(tableName).setRowKey(updateKey).execute(); + client.delete(tableName).setRowKey(appendKey).execute(); + client.delete(tableName).setRowKey(getKey).execute(); + } + } + + @Test + public void testQueryAndMutate() throws Exception { + // todo: only support in 3.x currently + if (ObTableClientTestUtil.isOBVersionGreaterEqualThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + String tableName = "test_varchar_table"; + String startKey = "k1"; + String endKey = "k2"; + ColumnValue updateVal = colVal("c2", "v2"); + try { + client.append(tableName).addScanRange(startKey, endKey).addMutateColVal(updateVal) + .execute(); + AuditRows auditRows = fetch_audit_rows(); + assertEquals(0, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + + client.insert(tableName).setRowKey(colVal("c1", startKey)).addMutateColVal(updateVal) + .execute(); + client.insert(tableName).setRowKey(colVal("c2", endKey)).addMutateColVal(updateVal) + .execute(); + + client.update(tableName).addScanRange(startKey, endKey).addMutateColVal(updateVal) + .execute(); + auditRows = fetch_audit_rows(); + assertEquals(2, auditRows.affected_rows); + // TODO: ODP do not serialize return_affected_entity_ in ObTableQueryAndMutate + // which causes return_affected_entity_ always be true + // This bug will be fixed soon by ODP guys + if (client.isOdpMode()) { + assertEquals(2, auditRows.return_rows); + } else { + assertEquals(0, auditRows.return_rows); + } + + client.delete(tableName).addScanRange(startKey, endKey).execute(); + auditRows = fetch_audit_rows(); + assertEquals(2, auditRows.affected_rows); + // TODO: ODP do not serialize return_affected_entity_ in ObTableQueryAndMutate + // which causes return_affected_entity_ always be true + // This bug will be fixed soon by ODP guys + if (client.isOdpMode()) { + assertEquals(2, auditRows.return_rows); + } else { + assertEquals(0, auditRows.return_rows); + } + + client.update(tableName).addScanRange(startKey, endKey).execute(); + auditRows = fetch_audit_rows(); + assertEquals(0, auditRows.affected_rows); + assertEquals(0, auditRows.return_rows); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + client.delete(tableName).addScanRange(startKey, endKey).execute(); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableTTLTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableTTLTest.java new file mode 100644 index 00000000..86c865cc --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableTTLTest.java @@ -0,0 +1,488 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc; + +import com.alipay.oceanbase.rpc.filter.ObCompareOp; +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.Map; + +import static com.alipay.oceanbase.rpc.filter.ObTableFilterFactory.compareVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.colVal; +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.row; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ObTableTTLTest { + ObTableClient client; + public static String tableName = "test_ttl_timestamp"; + public static String defaultValue = "hello world"; + public static String keyCol = "c1"; + public static String valueCol = "c2"; + public static String intCol = "c3"; + public static String expireCol = "expired_ts"; + + @Before + public void setup() throws Exception { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + this.client = obTableClient; + client.addRowKeyElement(tableName, new String[] { keyCol }); + } + + /** + CREATE TABLE `test_ttl_timestamp` ( + `c1` bigint NOT NULL, + `c2` varchar(20) DEFAULT NULL, + `c3` bigint DEFAULT NULL, + `expired_ts` timestamp, + PRIMARY KEY (`c1`)) TTL(expired_ts + INTERVAL 0 SECOND); + **/ + @Test + public void testQuery() throws Exception { + long[] keyIds = { 1L, 2L }; + try { + // 1. insert records with null expired_ts + for (long id : keyIds) { + client.insert(tableName).setRowKey(colVal(keyCol, id)) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, null)).execute(); + } + // 2. query all inserted records + QueryResultSet resultSet = client.query(tableName).addScanRange(keyIds[0], keyIds[1]) + .execute(); + Assert.assertEquals(resultSet.cacheSize(), keyIds.length); + + // 3. update the expired_ts + Timestamp curTs = new Timestamp(System.currentTimeMillis()); + curTs.setNanos(0); + client.update(tableName).setRowKey(colVal(keyCol, keyIds[0])) + .addMutateColVal(colVal(expireCol, curTs)).execute(); + + // 3. re-query all inserted records, the expired record won't be returned + resultSet = client.query(tableName).addScanRange(keyIds[0], keyIds[1]).execute(); + Assert.assertEquals(resultSet.cacheSize(), 1); + Assert.assertTrue(resultSet.next()); + Row row = resultSet.getResultRow(); + Assert.assertEquals(row.get(keyCol), keyIds[1]); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (long id : keyIds) { + client.delete(tableName).setRowKey(colVal(keyCol, id)).execute(); + } + } + } + + @Test + public void testQueryOffset() throws Exception { + long[] keyIds = { 1L, 2L, 3L, 4L }; + String tableName = "test_ttl_timestamp_5s"; + try { + Timestamp curTs = new Timestamp(System.currentTimeMillis()); + + // 1. insert records with current expired_ts + client.insert(tableName).setRowKey(colVal(keyCol, 1L)) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, curTs)).execute(); + + client.insert(tableName).setRowKey(colVal(keyCol, 2L)) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, curTs)).execute(); + + client.insert(tableName).setRowKey(colVal(keyCol, 3L)) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, null)).execute(); + + client.insert(tableName).setRowKey(colVal(keyCol, 4L)) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, null)).execute(); + + //+----+-------------+------+----------------------------+ + //| c1 | c2 | c3 | expired_ts | + //+----+-------------+------+----------------------------+ + //| 1 | hello world | NULL | xxx | + //| 2 | hello world | NULL | xxx | + //| 3 | hello world | NULL | NULL | + //| 4 | hello world | NULL | NULL | + //+----+-------------+------+----------------------------+ + Thread.sleep(6000); + + // 2. query offset-2 + QueryResultSet resultSet = client.query(tableName).addScanRange(keyIds[0], keyIds[3]) + .limit(2, 4).execute(); + Assert.assertEquals(resultSet.cacheSize(), 0); + + // 2. query offset-0 && limit-2 and c1 >= 0 + resultSet = client.query(tableName).addScanRange(keyIds[0], keyIds[3]).limit(0, 2) + .setFilter(compareVal(ObCompareOp.GE, keyCol, 0)).execute(); + Assert.assertEquals(resultSet.cacheSize(), 2); + + // 3. query offset-2 && limit-2 and c1 >= 0 + resultSet = client.query(tableName).addScanRange(keyIds[0], keyIds[3]).limit(2, 2) + .setFilter(compareVal(ObCompareOp.GE, keyCol, 0)).execute(); + Assert.assertEquals(resultSet.cacheSize(), 0); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + for (long id : keyIds) { + client.delete(tableName).setRowKey(colVal(keyCol, id)).execute(); + } + } + } + + @Test + public void test_single_operation() throws Exception { + long rowKey = 3L; + try { + long timeInMillis = System.currentTimeMillis() + 1000; + Timestamp ts = new Timestamp(timeInMillis); + ts.setNanos(0); + client.insert(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, ts)).execute(); + Thread.sleep(1000); + + // 1. get expired record, should return empty result + Map getRes = client.get(tableName, rowKey, null); + Assert.assertTrue(getRes.isEmpty()); + + // 2. insert new record, should success + MutationResult mutateRes = client.insert(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, null)).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + + // 3. update expired record, affected_rows should be 0 + ts = new Timestamp(System.currentTimeMillis()); + ts.setNanos(0); + mutateRes = client.update(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(expireCol, ts)).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + mutateRes = client.update(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(expireCol, null)).execute(); + assertEquals(0, mutateRes.getAffectedRows()); + + // 4. insertOrUpdate expired record, should insert success + ts = new Timestamp(System.currentTimeMillis() + 3000); + ts.setNanos(0); + mutateRes = client.insertOrUpdate(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(expireCol, ts)).addMutateColVal(colVal(intCol, 50L)) + .execute(); + assertEquals(1, mutateRes.getAffectedRows()); + getRes = client.get(tableName, rowKey, null); + Assert.assertEquals(null, getRes.get(valueCol)); + Assert.assertEquals(50L, getRes.get(intCol)); + // NOTE: this case failed with a small probability: the interval of insert and get exceed 3000ms + Assert.assertEquals(ts, getRes.get(expireCol)); + Thread.sleep(3000); + + // 5. increment a expired record, the behavior is same as increment a new record + mutateRes = client.increment(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(intCol, 100L)).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + getRes = client.get(tableName, rowKey, null); + Assert.assertEquals(null, getRes.get(expireCol)); + Assert.assertEquals(null, getRes.get(valueCol)); + Assert.assertEquals(100L, getRes.get(intCol)); + + // 6. append a expired record, the behavior is same as append a new record + ts = new Timestamp(System.currentTimeMillis()); + ts.setNanos(0); + mutateRes = client.update(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, ts)).execute(); + String appendVal = "how are u"; + mutateRes = client.append(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(valueCol, appendVal)).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + getRes = client.get(tableName, rowKey, null); + Assert.assertEquals(null, getRes.get(intCol)); + Assert.assertEquals(appendVal, getRes.get(valueCol)); + Assert.assertEquals(null, getRes.get(expireCol)); + + // 7. replace + ts = new Timestamp(System.currentTimeMillis()); + ts.setNanos(0); + mutateRes = client.update(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, ts)).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + mutateRes = client.replace(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(expireCol, null)) + .addMutateColVal(colVal(valueCol, defaultValue)).execute(); + assertEquals(2, mutateRes.getAffectedRows()); + getRes = client.get(tableName, rowKey, null); + Assert.assertEquals(null, getRes.get(intCol)); + Assert.assertEquals(defaultValue, getRes.get(valueCol)); + Assert.assertEquals(null, getRes.get(expireCol)); + + // 8. delete + ts = new Timestamp(System.currentTimeMillis()); + ts.setNanos(0); + mutateRes = client.update(tableName).setRowKey(colVal(keyCol, rowKey)) + .addMutateColVal(colVal(expireCol, ts)).execute(); + assertEquals(1, mutateRes.getAffectedRows()); + mutateRes = client.delete(tableName).setRowKey(colVal(keyCol, rowKey)).execute(); + assertEquals(0, mutateRes.getAffectedRows()); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + client.delete(tableName).addScanRange(rowKey, rowKey).execute(); + } + } + + // 1. query and update + // 2. query and increment + // 3. query and append + // 4. query and delete + @Test + public void testQueryAndMutate() throws Exception { + long[] keyIds = { 5L, 6L }; + + try { + long timeInMillis = System.currentTimeMillis(); + Timestamp curTs = new Timestamp(timeInMillis); + curTs.setNanos(0); + // 1. insert two records, one expired, one unexpired + MutationResult res = client.insert(tableName).setRowKey(colVal(keyCol, keyIds[0])) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, null)).execute(); + Assert.assertEquals(1, res.getAffectedRows()); + res = client.insert(tableName).setRowKey(colVal(keyCol, keyIds[1])) + .addMutateColVal(colVal(valueCol, defaultValue)) + .addMutateColVal(colVal(expireCol, curTs)).execute(); + Assert.assertEquals(1, res.getAffectedRows()); + + // 2. query and update all columns + res = client.update(tableName).addScanRange(keyIds[0], keyIds[1]) + .addMutateColVal(colVal(valueCol, "hello")).addMutateColVal(colVal(intCol, 100L)) + .execute(); + Assert.assertEquals(1, res.getAffectedRows()); + + QueryResultSet resultSet = client.query(tableName).addScanRange(keyIds[0], keyIds[1]) + .execute(); + Assert.assertEquals(resultSet.cacheSize(), 1); + resultSet.next(); + Row row = resultSet.getResultRow(); + Assert.assertEquals(row.get(keyCol), keyIds[0]); + Assert.assertEquals(row.get(intCol), 100L); + Assert.assertEquals(row.get(valueCol), "hello"); + Assert.assertEquals(1, res.getAffectedRows()); + + // 3. query and increment the int column to 200 + res = client.increment(tableName).addScanRange(keyIds[0], keyIds[1]) + .addMutateColVal(colVal(intCol, 100L)).execute(); + Assert.assertEquals(1, res.getAffectedRows()); + resultSet = client.query(tableName).addScanRange(keyIds[0], keyIds[1]).execute(); + Assert.assertEquals(resultSet.cacheSize(), 1); + resultSet.next(); + row = resultSet.getResultRow(); + Assert.assertEquals(row.get(keyCol), keyIds[0]); + Assert.assertEquals(row.get(intCol), 200L); + + // 4. query and append the value column to "hello world!" + res = client.append(tableName).addScanRange(keyIds[0], keyIds[1]) + .addMutateColVal(colVal(valueCol, ", world!")).execute(); + Assert.assertEquals(1, res.getAffectedRows()); + resultSet = client.query(tableName).addScanRange(keyIds[0], keyIds[1]).execute(); + Assert.assertEquals(resultSet.cacheSize(), 1); + resultSet.next(); + row = resultSet.getResultRow(); + Assert.assertEquals(row.get(keyCol), keyIds[0]); + Assert.assertEquals(row.get(valueCol), "hello, world!"); + + // 5. query and delete + res = client.delete(tableName).addScanRange(keyIds[0], keyIds[1]).execute(); + Assert.assertEquals(1, res.getAffectedRows()); + + resultSet = client.query(tableName).addScanRange(keyIds[0], keyIds[1]).execute(); + Assert.assertEquals(resultSet.cacheSize(), 0); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } finally { + client.delete(tableName).addScanRange(keyIds[0], keyIds[1]).execute(); + } + } + + @Test + public void test_ttl_table_query() throws SQLException { + /* + CREATE TABLE `table_ttl_00` ( + `name` varchar(512) NOT NULL, + `pk` varchar(512) NOT NULL, + `adiu` varchar(512) NOT NULL DEFAULT '', + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name_v` varchar(20) GENERATED ALWAYS AS (substr(`name`,1,5)) VIRTUAL, + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`adiu`, `pk`), + KEY `idx_adiu` (`adiu`, `pk`, `name`) LOCAL, + KEY `idx_pk` (`pk`) GLOBAL + ) TTL (gmt_create + INTERVAL 300 SECOND) partition by key(adiu) partitions 8; + */ + String TABLE_NAME = "table_ttl_00"; + try { + // prepare data + int recordCount = 2; + long cur_ts = System.currentTimeMillis(); + Timestamp cur_time = new Timestamp(cur_ts); + cur_time.setNanos(0); + Timestamp expire_time = new Timestamp(cur_ts - 301000); + expire_time.setNanos(0); + for (int i = 0; i < recordCount; i++) { + Timestamp ts; + if (i % 2 == 0) { + ts = cur_time; + } else { + ts = expire_time; + } + client.insert(TABLE_NAME) + .setRowKey(row(colVal("adiu", "adiu_" + i), colVal("pk", "pk_" + i))) + .addMutateRow(row(colVal("name", i + "_name"), colVal("gmt_create", ts))) + .execute(); + } + + // primary key: query with select all columns + int count = 0; + QueryResultSet res = client.query(TABLE_NAME) + .addScanRange(new Object[] { "adiu_0", "pk_0" }, new Object[] { "adiu_9", "pk_9" }) + .execute(); + while (res.next()) { + assertEquals(6, res.getRow().size()); + count++; + } + assertEquals(recordCount / 2, count); + + // primary key: query with select partial columns, without ttl column + count = 0; + res = client.query(TABLE_NAME) + .addScanRange(new Object[] { "adiu_0", "pk_0" }, new Object[] { "adiu_9", "pk_9" }) + .select("name", "pk").execute(); + while (res.next()) { + assertEquals(2, res.getRow().size()); + count++; + } + assertEquals(recordCount / 2, count); + + // primary key: query with select partial columns, with ttl column + res = client.query(TABLE_NAME) + .addScanRange(new Object[] { "adiu_0", "pk_0" }, new Object[] { "adiu_9", "pk_9" }) + .select("name", "pk", "gmt_create").execute(); + count = 0; + while (res.next()) { + assertEquals(3, res.getRow().size()); + count++; + } + assertEquals(recordCount / 2, count); + + // local index: query with select all columns + res = client + .query(TABLE_NAME) + .setScanRangeColumns(new String[] {"adiu", "pk", "name"}) + .addScanRange(new Object[] { "adiu_0", "pk_0", "0_name" }, + new Object[] { "adiu_9", "pk_9", "9_name" }).indexName("idx_adiu").execute(); + count = 0; + while (res.next()) { + assertEquals(6, res.getRow().size()); + count++; + } + assertEquals(recordCount / 2, count); + + // local index: query with select partial columns, without ttl columns + res = client + .query(TABLE_NAME) + .setScanRangeColumns(new String[] {"adiu", "pk", "name"}) + .addScanRange(new Object[] { "adiu_0", "pk_0", "0_name" }, + new Object[] { "adiu_9", "pk_9", "9_name" }).indexName("idx_adiu") + .select("adiu", "pk").execute(); + count = 0; + while (res.next()) { + assertEquals(2, res.getRow().size()); + count++; + } + assertEquals(recordCount / 2, count); + + // local index: query with select partial columns, without ttl columns + res = client + .query(TABLE_NAME) + .setScanRangeColumns("adiu", "pk", "name") + .addScanRange(new Object[] { "adiu_0", "pk_0", "0_name" }, + new Object[] { "adiu_9", "pk_9", "9_name" }).indexName("idx_adiu") + .select("adiu", "gmt_create").execute(); + count = 0; + while (res.next()) { + assertEquals(2, res.getRow().size()); + count++; + } + assertEquals(recordCount / 2, count); + + // global index: query with select all columns + res = client.query(TABLE_NAME).setScanRangeColumns("pk") + .addScanRange(new Object[] { "pk_0" }, new Object[] { "pk_9" }).indexName("idx_pk") + .execute(); + count = 0; + while (res.next()) { + assertEquals(6, res.getRow().size()); + count++; + } + assertEquals(recordCount / 2, count); + + // global index: query with select partial columns, without ttl columns + res = client.query(TABLE_NAME).setScanRangeColumns("pk") + .addScanRange(new Object[] { "pk_0" }, new Object[] { "pk_9" }).indexName("idx_pk") + .select("pk").execute(); + count = 0; + while (res.next()) { + assertEquals(1, res.getRow().size()); + count++; + } + assertEquals(recordCount / 2, count); + + // global index: query with select partial columns, with ttl columns + res = client.query(TABLE_NAME).setScanRangeColumns("pk") + .addScanRange(new Object[] { "pk_0" }, new Object[] { "pk_9" }).indexName("idx_pk") + .select("pk", "gmt_create").execute(); + count = 0; + while (res.next()) { + assertEquals(2, res.getRow().size()); + count++; + } + assertEquals(recordCount / 2, count); + + } catch (Exception ex) { + ex.printStackTrace(); + assertTrue(false); + } finally { + ObTableClientTestUtil.getConnection().createStatement() + .execute("truncate table " + TABLE_NAME); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObWeakReadConsistencyTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObWeakReadConsistencyTest.java index f2774f40..8a39a377 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObWeakReadConsistencyTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObWeakReadConsistencyTest.java @@ -20,6 +20,7 @@ import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; import com.alipay.oceanbase.rpc.location.model.*; import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import com.alipay.oceanbase.rpc.threadlocal.ThreadLocalMap; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; import com.alipay.oceanbase.rpc.util.ZoneUtil; import org.junit.*; @@ -32,6 +33,7 @@ public class ObWeakReadConsistencyTest { private static final String weakParamUrl = paramUrl + "&read_consistency=weak&ob_route_policy=follower_first"; private static final int dataSetSize = 10; + private static String testIdc = "dev"; protected ObTableClient client; public static void initZoneClient() { @@ -45,59 +47,68 @@ public static ObTableClient getObTableClient(String paramUrl) throws Exception { obTableClient.setFullUserName(fullUserName); obTableClient.setParamURL(paramUrl); obTableClient.setPassword(password); - //obTableClient.setCurrentIDC(testIdc); + obTableClient.setCurrentIDC(testIdc); obTableClient.setSysUserName(ObTableClientTestUtil.PROXY_SYS_USER_NAME); - obTableClient.setEncSysPassword(ObTableClientTestUtil.PROXY_SYS_USER_ENC_PASSWORD); + obTableClient.setSysPassword(ObTableClientTestUtil.PROXY_SYS_USER_PASSWORD); obTableClient.init(); return obTableClient; } @BeforeClass public static void init() throws Exception { - cleanup(); - ObTableClient client = getObTableClient(paramUrl); - for (int i = 0; i < dataSetSize; i++) { - String key = "abc-" + i; - String val = "xyz-" + i; - client.insert("test_varchar_table", key, new String[] { "c2" }, new String[] { val }); - } + // TODO: this test need refactored + // ThreadLocalMap.setReadConsistency(ObReadConsistency.WEAK); + // cleanup(); + // ObTableClient client = getObTableClient(paramUrl); + // for (int i = 0; i < dataSetSize; i++) { + // String key = "abc-" + i; + // String val = "xyz-" + i; + // client.insert("test_varchar_table", key, new String[] { "c2" }, new String[] { val }); + // } } @AfterClass public static void cleanup() throws Exception { - ObTableClient client = getObTableClient(paramUrl); - for (int i = 0; i < dataSetSize; i++) { - String key = "abc-" + i; - client.delete("test_varchar_table", key); - } + // TODO: this test need refactored + return; + // ObTableClient client = getObTableClient(paramUrl); + // for (int i = 0; i < dataSetSize; i++) { + // String key = "abc-" + i; + // client.delete("test_varchar_table", key); + // } } @Before public void setup() throws Exception { - client = getObTableClient(weakParamUrl); + // client = getObTableClient(weakParamUrl); } @After public void teardown() throws Exception { - client.close(); + // client.close(); } @Test public void testReadConsistencySetting() { - Assert.assertEquals(client.getReadConsistency(), ObReadConsistency.WEAK); - Assert.assertEquals(client.getObRoutePolicy(), ObRoutePolicy.FOLLOWER_FIRST); + // Assert.assertEquals(client.getReadConsistency(), ObReadConsistency.WEAK); + // Assert.assertEquals(client.getObRoutePolicy(), ObRoutePolicy.FOLLOWER_FIRST); } @Test public void testGetWithWeakRead() throws Exception { + // skip test now + if (true) { + return; + } + for (int i = 0; i < dataSetSize; i++) { String key = "abc-" + i; String val = "xyz-" + i; try { Map res = client.get("test_varchar_table", key, new String[] { "c1", "c2" }); - Assert.fail("failed to get with weak read"); + Assert.assertEquals(val, res.get("c2")); } catch (ObTableUnexpectedException e) { - Assert.assertEquals(ResultCodes.OB_NOT_SUPPORTED.errorCode, e.getErrorCode()); + Assert.fail("failed to get with weak read"); } } Assert.assertFalse(client.getReadConsistency().isStrong()); @@ -105,6 +116,10 @@ public void testGetWithWeakRead() throws Exception { @Test public void testZoneIdc() { + // skip test now + if (true) { + return; + } Assert.assertEquals("dev", ZoneUtil.getCurrentIDC()); } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/bolt/ObTableClientTestBase.java b/src/test/java/com/alipay/oceanbase/rpc/bolt/ObTableClientTestBase.java index e34ae3af..c4ebedd0 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/bolt/ObTableClientTestBase.java +++ b/src/test/java/com/alipay/oceanbase/rpc/bolt/ObTableClientTestBase.java @@ -20,18 +20,24 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.exception.ObTableDuplicateKeyException; import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.location.model.partition.ObPair; import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObHTableFilter; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryRequest; import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.table.ObTable; +import com.alipay.oceanbase.rpc.table.ObTableParam; import com.alipay.oceanbase.rpc.table.api.Table; import com.alipay.oceanbase.rpc.table.api.TableBatchOps; import com.alipay.oceanbase.rpc.table.api.TableQuery; +import org.junit.After; import org.junit.Assert; import org.junit.Test; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Vector; import static org.junit.Assert.*; import static org.junit.Assert.assertNotNull; @@ -40,17 +46,35 @@ public abstract class ObTableClientTestBase { public Table client; + public void setClient(Table client) { + this.client = client; + } + + public boolean skipObTableTest() { + return this instanceof ObTableTest; + } + + @After + public void close() throws Exception { + if (null != this.client && this.client instanceof ObTableClient) { + ((ObTableClient) this.client).close(); + } + } + @Test public void test_batch() throws Exception { - /* - * CREATE TABLE `test_varchar_table` ( `c1` varchar(20) NOT NULL, `c2` varchar(20) DEFAULT NULL, PRIMARY KEY (`c1`) ) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = 3 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 10 - * */ + */ + if (this.getClass().getSimpleName().equals("ObTableTest")) { + // could not get tableId from ObTable + return; + } + for (int i = 0; i < 100; i++) { TableBatchOps batchOps = client.batch("test_varchar_table"); @@ -103,6 +127,12 @@ public void test_blob_all() throws Exception { PRIMARY KEY (`c1`) ) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = 3 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 10 * */ + + // NOTE: Use the default test_varchar_table ObTable to insert data for test_blob_table will causes OB_NOT_MASTER error + // when their partition leader is not in the same observer + if (client instanceof ObTable) { + return; + } try { test_blob_insert(); long start = System.currentTimeMillis(); @@ -134,11 +164,17 @@ public void test_longblob_all() throws Exception { PRIMARY KEY (`c1`) ) DEFAULT CHARSET = utf8mb4 COMPRESSION = 'lz4_1.0' REPLICA_NUM = 3 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 10 * */ + // NOTE: Use the default test_varchar_table ObTable to insert data for test_blob_table will causes OB_NOT_MASTER error + // when their partition leader is not in the same observer + if (client instanceof ObTable) { + return; + } try { test_longblob_insert(); long start = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { test_longblob_get(); + System.out.println(i); } System.err.println("cost: " + (System.currentTimeMillis() - start)); test_longblob_update(); @@ -159,10 +195,15 @@ public void test_varchar_exceptions() throws Exception { try { client.insert("not_exist_table", "foo", new String[] { "c2" }, new String[] { "bar" }); } catch (ObTableException ex) { + ex.printStackTrace(); exception = ex; } assertNotNull(exception); - assertEquals(ResultCodes.OB_ERR_UNKNOWN_TABLE.errorCode, exception.getErrorCode()); + if ((client instanceof ObTableClient) && ((ObTableClient) client).isOdpMode()) { + assertEquals(ResultCodes.OB_TABLE_NOT_EXIST.errorCode, exception.getErrorCode()); + } else { + assertEquals(ResultCodes.OB_ERR_UNKNOWN_TABLE.errorCode, exception.getErrorCode()); + } exception = null; try { @@ -172,8 +213,9 @@ public void test_varchar_exceptions() throws Exception { exception = ex; } assertNotNull(exception); - assertEquals(ResultCodes.OB_ERR_COLUMN_NOT_FOUND.errorCode, exception.getErrorCode()); - assertTrue(exception.getMessage().contains("column not found")); + System.out.printf("exception msg：%s\n", exception.getMessage()); + assertEquals(ResultCodes.OB_ERR_BAD_FIELD_ERROR.errorCode, exception.getErrorCode()); + assertTrue(exception.getMessage().contains("Unknown column")); exception = null; try { @@ -183,8 +225,12 @@ public void test_varchar_exceptions() throws Exception { exception = ex; } assertNotNull(exception); - assertEquals(ResultCodes.OB_OBJ_TYPE_ERROR.errorCode, exception.getErrorCode()); - assertTrue(exception.getMessage().contains("rowkey/column type not match")); + System.out.printf("exception msg：%s\n", exception.getMessage()); + assertEquals(ResultCodes.OB_KV_COLUMN_TYPE_NOT_MATCH.errorCode, exception.getErrorCode()); + assertTrue(exception + .getMessage() + .contains( + "Column type for 'c1' not match, schema column type is 'VARCHAR', input column type is 'INT'")); exception = null; try { @@ -194,8 +240,12 @@ public void test_varchar_exceptions() throws Exception { exception = ex; } assertNotNull(exception); - assertEquals(ResultCodes.OB_OBJ_TYPE_ERROR.errorCode, exception.getErrorCode()); - assertTrue(exception.getMessage().contains("rowkey/column type not match")); + System.out.printf("exception msg：%s\n", exception.getMessage()); + assertEquals(ResultCodes.OB_KV_COLUMN_TYPE_NOT_MATCH.errorCode, exception.getErrorCode()); + assertTrue(exception + .getMessage() + .contains( + "Column type for 'c2' not match, schema column type is 'VARCHAR', input column type is 'INT'")); exception = null; try { @@ -205,8 +255,9 @@ public void test_varchar_exceptions() throws Exception { exception = ex; } assertNotNull(exception); + System.out.printf("exception msg：%s\n", exception.getMessage()); assertEquals(ResultCodes.OB_BAD_NULL_ERROR.errorCode, exception.getErrorCode()); - assertTrue(exception.getMessage().contains("column doesn't have a default value")); + assertTrue(exception.getMessage().contains("Column 'c2' cannot be null")); // TODO timeout } @@ -216,11 +267,11 @@ private void test_varchar_insert() throws Exception { assertEquals(1L, client.insert("test_varchar_table", "foo", new String[] { "c2" }, new String[] { "bar" })); - ObTableDuplicateKeyException ex = null; + ObTableException ex = null; try { assertEquals(1L, client.insert("test_varchar_table", "foo", new String[] { "c2" }, new String[] { "baz" })); - } catch (ObTableDuplicateKeyException t) { + } catch (ObTableException t) { ex = t; } assertNotNull(ex); @@ -228,7 +279,7 @@ private void test_varchar_insert() throws Exception { try { assertEquals(1L, client.insert("test_varchar_table", "foo", new String[] { "c2" }, new String[] { "bar" })); - } catch (ObTableDuplicateKeyException t) { + } catch (ObTableException t) { ex = t; } assertNotNull(ex); @@ -241,13 +292,13 @@ private void test_blob_insert() throws Exception { client.insert("test_blob_table", "foo", new String[] { "c2" }, new Object[] { "bar".getBytes() })); - ObTableDuplicateKeyException ex = null; + ObTableException ex = null; try { assertEquals( 1L, client.insert("test_blob_table", "foo", new String[] { "c2" }, new Object[] { "baz".getBytes() })); - } catch (ObTableDuplicateKeyException t) { + } catch (ObTableException t) { ex = t; } assertNotNull(ex); @@ -257,7 +308,7 @@ private void test_blob_insert() throws Exception { 1L, client.insert("test_blob_table", "foo", new String[] { "c2" }, new Object[] { "bar".getBytes() })); - } catch (ObTableDuplicateKeyException t) { + } catch (ObTableException t) { ex = t; } assertNotNull(ex); @@ -274,11 +325,11 @@ private void test_longblob_insert() throws Exception { client.insert("test_longblob_table", "foo", new String[] { "c2" }, new Object[] { "bar".getBytes() })); - ObTableDuplicateKeyException ex = null; + ObTableException ex = null; try { assertEquals(1L, client.insert("test_longblob_table", "foo", new String[] { "c2" }, new Object[] { "baz".getBytes() })); - } catch (ObTableDuplicateKeyException t) { + } catch (ObTableException t) { ex = t; } assertNotNull(ex); @@ -286,7 +337,7 @@ private void test_longblob_insert() throws Exception { try { assertEquals(1L, client.insert("test_longblob_table", "foo", new String[] { "c2" }, new Object[] { "bar".getBytes() })); - } catch (ObTableDuplicateKeyException t) { + } catch (ObTableException t) { ex = t; } assertNotNull(ex); @@ -511,139 +562,418 @@ private void test_longblob_replace() throws Exception { assertEquals("baz", new String((byte[]) values.get("c2"))); } - /** - * 单 key 的方式，例如 - * scan[{"123"}, {"567"}) - * @throws Exception - */ @Test - public void test_limit_query_1() throws Exception { + public void test_limit_query() throws Exception { + if (skipObTableTest()) { + return; + } + Object[] c1 = new Object[] { "123", "124", "234", "456", "567" }; Object[] c2 = new Object[] { "123c2", "124c2", "234c2", "456c2", "567c2" }; + if ((client instanceof ObTableClient) && ((ObTableClient) client).isOdpMode()) { + // TODO: support stream result + } else { + try { + for (int i = 0; i < 5; i++) { + client.insert("test_varchar_table", c1[i], new String[] { "c2" }, + new Object[] { c2[i] }); + } + try { + TableQuery tableQuery = client.query("test_varchar_table"); + QueryResultSet result = tableQuery.select("c2").setBatchSize(1) + .addScanRange("123", true, "567", true).execute(); + fail(); + } catch (ObTableException e) { + assertTrue(true); + } + + // 123 <= xxx <= 567 + TableQuery tableQuery = client.query("test_varchar_table"); + QueryResultSet result = tableQuery.select("c1", "c2").setBatchSize(1) + .addScanRange("123", true, "567", true).asyncExecute(); + for (int i = 0; i < 5; i++) { + Assert.assertTrue(result.next()); + Map value = result.getRow(); + assertEquals(value.get("c2"), c2[i]); + } + Assert.assertFalse(result.next()); + + // 123 <= xxx < 567 + tableQuery = client.query("test_varchar_table"); + result = tableQuery.select("c1", "c2").setBatchSize(1) + .addScanRange("123", true, "567", false).asyncExecute(); + for (int i = 0; i < 4; i++) { + Assert.assertTrue(result.next()); + assertEquals(0, result.cacheSize()); + Map value = result.getRow(); + assertEquals(value.get("c2"), c2[i]); + } + Assert.assertFalse(result.next()); + result.close(); + } finally { + for (int i = 0; i < 5; i++) { + client.delete("test_varchar_table", c1[i]); + } + } + } + } + + @Test + public void test_async_query() throws Exception { + /* + * CREATE TABLE `test_varchar_table` ( + `c1` varchar(20) NOT NULL, + `c2` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + )*/ + if (skipObTableTest()) { + return; + } try { + client.delete("test_varchar_table", "125"); + client.insert("test_varchar_table", "125", new String[] { "c2" }, + new Object[] { "123c1" }); + TableQuery tableQuery = client.query("test_varchar_table"); + QueryResultSet resultSet = tableQuery.select("c2").primaryIndex() + .addScanRange("123", "567").asyncExecute(); + assertEquals(1, resultSet.cacheSize()); + client.delete("test_varchar_table", "125"); + + client.insert("test_varchar_table", "123", new String[] { "c2" }, + new Object[] { "123c2" }); + client.insert("test_varchar_table", "124", new String[] { "c2" }, + new Object[] { "124c2" }); + client.insert("test_varchar_table", "234", new String[] { "c2" }, + new Object[] { "234c2" }); + client.insert("test_varchar_table", "456", new String[] { "c2" }, + new Object[] { "456c2" }); + client.insert("test_varchar_table", "567", new String[] { "c2" }, + new Object[] { "567c2" }); + + tableQuery = client.query("test_varchar_table"); + tableQuery.setMaxResultSize(100000); + tableQuery.clear(); + + // >= 123 && <= 567 + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("123", "567") + .asyncExecute(); + assertEquals(5, resultSet.cacheSize()); for (int i = 0; i < 5; i++) { - client.insert("test_varchar_table", c1[i], new String[] { "c2" }, - new Object[] { c2[i] }); + Assert.assertTrue(resultSet.next()); + Map value = resultSet.getRow(); + switch (i) { + case 0: + assertEquals("123c2", value.get("c2")); + break; + case 1: + assertEquals("124c2", value.get("c2")); + break; + case 2: + assertEquals("234c2", value.get("c2")); + break; + case 3: + assertEquals("456c2", value.get("c2")); + break; + case 4: + assertEquals("567c2", value.get("c2")); + break; + } } - // 123 <= xxx <= 567 - TableQuery tableQuery = client.queryByBatch("test_varchar_table"); - QueryResultSet result = tableQuery.setKeys("c1").select("c2").setBatchSize(1) - .addScanRange("123", true, "567", true).execute(); - for (int i = 0; i < 5; i++) { - Assert.assertTrue(result.next()); - Map value = result.getRow(); - assertEquals(value.get("c2"), c2[i]); + + // >= 123 && <= 123 + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("123", "123") + .asyncExecute(); + assertEquals(1, resultSet.cacheSize()); + Assert.assertTrue(resultSet.next()); + Map v = resultSet.getRow(); + assertEquals("123c2", v.get("c2")); + + // >= 124 && <= 456 + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("124", "456") + .asyncExecute(); + for (int i = 0; i < 3; i++) { + Assert.assertTrue(resultSet.next()); + Map value = resultSet.getRow(); + switch (i) { + case 0: + assertEquals("124c2", value.get("c2")); + break; + case 1: + assertEquals("234c2", value.get("c2")); + break; + case 2: + assertEquals("456c2", value.get("c2")); + break; + } + } + + // > 123 && < 567 + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex() + .addScanRange(new Object[] { "123" }, false, new Object[] { "567" }, false) + .setBatchSize(1).asyncExecute(); + for (int i = 0; i < 3; i++) { + Assert.assertTrue(resultSet.next()); + Map value = resultSet.getRow(); + switch (i) { + case 0: + assertEquals("124c2", value.get("c2")); + break; + case 1: + assertEquals("234c2", value.get("c2")); + break; + case 2: + assertEquals("456c2", value.get("c2")); + break; + } } - Assert.assertFalse(result.next()); - // 123 <= xxx < 567 - tableQuery = client.queryByBatch("test_varchar_table"); - result = tableQuery.setKeys("c1").select("c1", "c2").setBatchSize(1) - .addScanRange("123", true, "567", false).execute(); + // > 123 && <= 567 + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex() + .addScanRange(new Object[] { "123" }, false, new Object[] { "567" }, true) + .setBatchSize(2).asyncExecute(); for (int i = 0; i < 4; i++) { - Assert.assertTrue(result.next()); - assertEquals(0, result.cacheSize()); - Map value = result.getRow(); - assertEquals(value.get("c2"), c2[i]); + Assert.assertTrue(resultSet.next()); + Map value = resultSet.getRow(); + switch (i) { + case 0: + assertEquals("124c2", value.get("c2")); + break; + case 1: + assertEquals("234c2", value.get("c2")); + break; + case 2: + assertEquals("456c2", value.get("c2")); + break; + case 3: + assertEquals("567c2", value.get("c2")); + break; + } } - Assert.assertFalse(result.next()); - result.close(); - } finally { - for (int i = 0; i < 5; i++) { - client.delete("test_varchar_table", c1[i]); + + // >= 123 && < 567 + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex() + .addScanRange(new Object[] { "123" }, true, new Object[] { "567" }, false) + .setBatchSize(2).asyncExecute(); + for (int i = 0; i < 4; i++) { + Assert.assertTrue(resultSet.next()); + Map value = resultSet.getRow(); + switch (i) { + case 0: + assertEquals("123c2", value.get("c2")); + break; + case 1: + assertEquals("124c2", value.get("c2")); + break; + case 2: + assertEquals("234c2", value.get("c2")); + break; + case 3: + assertEquals("456c2", value.get("c2")); + break; + } } - } - } - @Test - public void test_limit_query_2() throws Exception { - TableQuery tableQuery = client.queryByBatch("test_varchar_table"); - TableQuery tableQuery2 = client.queryByBatch("test_varchar_table"); - tableQuery.setOperationTimeout(100000); - assertNotNull(tableQuery.getObTableQuery()); - tableQuery.setEntityType(new ObTableQueryRequest().getEntityType()); - assertNotNull(tableQuery.getEntityType()); - assertEquals("test_varchar_table", tableQuery.getTableName()); - tableQuery.addScanRange("1", "2"); - tableQuery2.addScanRangeStartsWith("1"); - tableQuery2.addScanRangeEndsWith("2"); - assertEquals(1, tableQuery.getObTableQuery().getKeyRanges().size()); - assertEquals(2, tableQuery2.getObTableQuery().getKeyRanges().size()); + // >= 12 && <= 126 + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("12", "126") + .setBatchSize(10).asyncExecute(); + assertEquals(2, resultSet.cacheSize()); + for (int i = 0; i < 2; i++) { + Assert.assertTrue(resultSet.next()); + Map value = resultSet.getRow(); + switch (i) { + case 0: + assertEquals("123c2", value.get("c2")); + break; + case 1: + assertEquals("124c2", value.get("c2")); + break; + } + } - try { - tableQuery.scanOrder(true); - fail(); - } catch (Exception e) { - assertTrue(true); - } + // (>=12 && <=126) || (>="456" && <="567") + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("12", "126") + .addScanRange("456", "567").asyncExecute(); + assertEquals(4, resultSet.cacheSize()); + for (int i = 0; i < 4; i++) { + Assert.assertTrue(resultSet.next()); + Map value = resultSet.getRow(); + switch (i) { + case 0: + assertEquals("123c2", value.get("c2")); + break; + case 1: + assertEquals("124c2", value.get("c2")); + break; + case 2: + assertEquals("456c2", value.get("c2")); + break; + case 3: + assertEquals("567c2", value.get("c2")); + break; + } + } - try { - tableQuery.indexName("test"); - fail(); - } catch (Exception e) { - assertTrue(true); - } + // (>=124 && <=124) + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("124", "124") + .setBatchSize(1).asyncExecute(); + for (int i = 0; i < 1; i++) { + Assert.assertTrue(resultSet.next()); + Map value = resultSet.getRow(); + switch (i) { + case 0: + assertEquals("124c2", value.get("c2")); + break; + } + } - try { - tableQuery.primaryIndex(); - fail(); - } catch (Exception e) { - assertTrue(true); - } + // (>=124 && <=123) + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("124", "123") + .setBatchSize(10).asyncExecute(); + assertEquals(0, resultSet.cacheSize()); - try { - tableQuery.filterString("111"); - fail(); - } catch (Exception e) { - assertTrue(true); - } + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("12", "126") + .addScanRange("456", "567").setBatchSize(1).asyncExecute(); + for (int i = 0; i < 1; i++) { + Assert.assertTrue(resultSet.next()); + Map value = resultSet.getRow(); + switch (i) { + case 0: + assertEquals("123c2", value.get("c2")); + break; + } + } + resultSet.close(); - try { - tableQuery.setHTableFilter(new ObHTableFilter()); - fail(); - } catch (Exception e) { - assertTrue(true); - } + try { + resultSet.next(); + fail(); + } catch (IllegalStateException e) { + Assert.assertTrue(e.getMessage().contains("closed")); + } - try { - tableQuery.limit(10); - fail(); - } catch (Exception e) { - assertTrue(true); - } + // test async close + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("123", "567") + .setBatchSize(2).asyncExecute(); + assertEquals(2, resultSet.cacheSize()); + resultSet.close(); + try { + resultSet.next(); + fail(); + } catch (IllegalStateException e) { + Assert.assertTrue(e.getMessage().contains("query stream result is closed")); + } - try { - tableQuery.limit(10, 10); - fail(); - } catch (Exception e) { - assertTrue(true); - } + // TODO: add test to check query timeout - try { - tableQuery.addScanRange(new Object[] { "1" }, new Object[] { "3" }).setKeys("c1", "c1"); - fail(); - } catch (Exception e) { - assertTrue(true); + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex().addScanRange("12", "126") + .addScanRange("456", "567").setOperationTimeout(3000).execute(); + assertEquals(4, resultSet.cacheSize()); + Assert.assertTrue(resultSet.next()); + Thread.sleep(2000); + resultSet.next(); + resultSet.close(); + + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex().scanOrder(true) + .addScanRangeStartsWith(new Object[] { "12" }).setOperationTimeout(3000).execute(); + assertEquals(5, resultSet.cacheSize()); + Assert.assertTrue(resultSet.next()); + Thread.sleep(2000); + resultSet.next(); + resultSet.close(); + + tableQuery.clear(); + resultSet = tableQuery.select("c2").primaryIndex().scanOrder(true) + .addScanRangeEndsWith(new Object[] { "126" }).setOperationTimeout(3000).execute(); + assertEquals(2, resultSet.cacheSize()); + Assert.assertTrue(resultSet.next()); + Thread.sleep(2000); + resultSet.next(); + resultSet.close(); + + tableQuery.clear(); + try { + tableQuery.select("c2").addScanRange("12", "126").addScanRange("456", "567") + .setBatchSize(1).execute(); + } catch (Exception e) { + if (e instanceof ObTableException) { + assertTrue(true); + } else { + System.out.println("Wrong Exception: " + e); + fail(); + } + } + } finally { + client.delete("test_varchar_table", "123"); + client.delete("test_varchar_table", "124"); + client.delete("test_varchar_table", "125"); + client.delete("test_varchar_table", "234"); + client.delete("test_varchar_table", "456"); + client.delete("test_varchar_table", "567"); } + } - try { - tableQuery.setKeys("c1", "c3").select("c2", "c1"); - fail(); - } catch (Exception e) { - assertTrue(true); + @Test + public void test_async_query_timeout() throws Exception { + /* + * CREATE TABLE `test_varchar_table` ( + `c1` varchar(20) NOT NULL, + `c2` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + )*/ + if (skipObTableTest()) { + return; } + try { + client.insert("test_varchar_table", "123", new String[] { "c2" }, + new Object[] { "123c2" }); + client.insert("test_varchar_table", "124", new String[] { "c2" }, + new Object[] { "124c2" }); + client.insert("test_varchar_table", "234", new String[] { "c2" }, + new Object[] { "234c2" }); + client.insert("test_varchar_table", "456", new String[] { "c2" }, + new Object[] { "456c2" }); + client.insert("test_varchar_table", "567", new String[] { "c2" }, + new Object[] { "567c2" }); + + TableQuery tableQuery = client.query("test_varchar_table"); + tableQuery.setMaxResultSize(100000); + tableQuery.clear(); - tableQuery.clear(); + // Query session will be delete after couple minutes, if everything goes well, server will survive :D + QueryResultSet resultSet = tableQuery.select("c2").primaryIndex().setBatchSize(1) + .addScanRange("123", "567").asyncExecute(); + } finally { + client.delete("test_varchar_table", "123"); + client.delete("test_varchar_table", "124"); + client.delete("test_varchar_table", "125"); + client.delete("test_varchar_table", "234"); + client.delete("test_varchar_table", "456"); + client.delete("test_varchar_table", "567"); + } } @Test public void test_query() throws Exception { /* * CREATE TABLE `test_varchar_table` ( - `c1` varchar(20) NOT NULL, - `c2` varchar(20) DEFAULT NULL, - PRIMARY KEY (`c1`) - )*/ + `c1` varchar(20) NOT NULL, + `c2` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + )*/ + if (skipObTableTest()) { + return; + } try { client.delete("test_varchar_table", "125"); client.insert("test_varchar_table", "125", new String[] { "c2" }, @@ -666,6 +996,9 @@ PRIMARY KEY (`c1`) new Object[] { "567c2" }); tableQuery = client.query("test_varchar_table"); + tableQuery.setMaxResultSize(100000); + tableQuery.clear(); + // >= 123 && <= 567 resultSet = tableQuery.select("c2").primaryIndex().addScanRange("123", "567").execute(); assertEquals(5, resultSet.cacheSize()); @@ -851,34 +1184,9 @@ PRIMARY KEY (`c1`) resultSet = tableQuery.select("c2").primaryIndex().addScanRange("124", "123").execute(); assertEquals(0, resultSet.cacheSize()); - tableQuery.clear(); resultSet = tableQuery.select("c2").primaryIndex().addScanRange("12", "126") - .addScanRange("456", "567").setBatchSize(1).execute(); - assertEquals(0, resultSet.cacheSize()); - for (int i = 0; i < 4; i++) { - Assert.assertTrue(resultSet.next()); - Map value = resultSet.getRow(); - switch (i) { - case 0: - assertEquals("123c2", value.get("c2")); - break; - case 1: - assertEquals("124c2", value.get("c2")); - break; - case 2: - assertEquals("456c2", value.get("c2")); - break; - case 3: - assertEquals("567c2", value.get("c2")); - break; - } - } - - Assert.assertFalse(resultSet.next()); - - resultSet = tableQuery.select("c2").primaryIndex().addScanRange("12", "126") - .addScanRange("456", "567").setBatchSize(1).execute(); - assertEquals(0, resultSet.cacheSize()); + .addScanRange("456", "567").execute(); + assertEquals(4, resultSet.cacheSize()); for (int i = 0; i < 1; i++) { Assert.assertTrue(resultSet.next()); Map value = resultSet.getRow(); @@ -897,25 +1205,12 @@ PRIMARY KEY (`c1`) Assert.assertTrue(e.getMessage().contains("closed")); } - tableQuery.clear(); - resultSet = tableQuery.select("c2").primaryIndex().addScanRange("12", "126") - .addScanRange("456", "567").setBatchSize(1).setOperationTimeout(1000).execute(); - assertEquals(0, resultSet.cacheSize()); - Assert.assertTrue(resultSet.next()); - Thread.sleep(2000); - try { - resultSet.next(); - fail(); - } catch (ObTableException e) { - // ob 1.x 版本和 ob 2.x 版本 errorCode 不一致 - Assert.assertTrue(e.getErrorCode() == ResultCodes.OB_TRANS_TIMEOUT.errorCode - || e.getErrorCode() == ResultCodes.OB_TRANS_ROLLBACKED.errorCode); - } + // TODO: add test to check query timeout tableQuery.clear(); resultSet = tableQuery.select("c2").primaryIndex().addScanRange("12", "126") - .addScanRange("456", "567").setBatchSize(1).setOperationTimeout(3000).execute(); - assertEquals(0, resultSet.cacheSize()); + .addScanRange("456", "567").setOperationTimeout(3000).execute(); + assertEquals(4, resultSet.cacheSize()); Assert.assertTrue(resultSet.next()); Thread.sleep(2000); resultSet.next(); @@ -923,9 +1218,8 @@ PRIMARY KEY (`c1`) tableQuery.clear(); resultSet = tableQuery.select("c2").primaryIndex().scanOrder(true) - .addScanRangeStartsWith(new Object[] { "12" }).setBatchSize(1) - .setOperationTimeout(3000).execute(); - assertEquals(0, resultSet.cacheSize()); + .addScanRangeStartsWith(new Object[] { "12" }).setOperationTimeout(3000).execute(); + assertEquals(5, resultSet.cacheSize()); Assert.assertTrue(resultSet.next()); Thread.sleep(2000); resultSet.next(); @@ -933,58 +1227,77 @@ PRIMARY KEY (`c1`) tableQuery.clear(); resultSet = tableQuery.select("c2").primaryIndex().scanOrder(true) - .addScanRangeEndsWith(new Object[] { "126" }).setBatchSize(1) - .setOperationTimeout(3000).execute(); - assertEquals(0, resultSet.cacheSize()); + .addScanRangeEndsWith(new Object[] { "126" }).setOperationTimeout(3000).execute(); + assertEquals(2, resultSet.cacheSize()); Assert.assertTrue(resultSet.next()); Thread.sleep(2000); resultSet.next(); resultSet.close(); + + tableQuery.clear(); + try { + tableQuery.select("c2").addScanRange("12", "126").addScanRange("456", "567") + .setBatchSize(1).execute(); + } catch (Exception e) { + if (e instanceof ObTableException) { + assertTrue(true); + } else { + System.out.println("Wrong Exception: " + e); + fail(); + } + } } finally { client.delete("test_varchar_table", "123"); client.delete("test_varchar_table", "124"); + client.delete("test_varchar_table", "125"); client.delete("test_varchar_table", "234"); client.delete("test_varchar_table", "456"); client.delete("test_varchar_table", "567"); } } - private void test_varchar_get_helper(String key, String value) throws Exception { - Map values = client.get("test_varchar_table", key, new String[] { "c2" }); + private void test_varchar_get_helper(ObTableClient obTableClient, String key, String value) + throws Exception { + Map values = obTableClient.get("test_varchar_table", key, + new String[] { "c2" }); assertNotNull(values); assertEquals(value, values.get("c2")); } - private void test_varchar_insert_helper(String key, String value) throws Exception { - long affectedRows = client.insert("test_varchar_table", key, new String[] { "c2" }, + private void test_varchar_insert_helper(ObTableClient obTableClient, String key, String value) + throws Exception { + long affectedRows = obTableClient.insert("test_varchar_table", key, new String[] { "c2" }, new String[] { value }); assertEquals(1L, affectedRows); } - public void test_varchar_helper(String prefix, int count) throws Exception { + public void test_varchar_helper(ObTableClient obTableClient, String prefix, int count) + throws Exception { String keyPrefix = "K" + prefix; String valPrefix = "V" + prefix; try { for (int i = 0; i < count; i++) { - test_varchar_insert_helper(keyPrefix + i, valPrefix + i); + test_varchar_insert_helper(obTableClient, keyPrefix + i, valPrefix + i); } for (int i = 0; i < count; i++) { - test_varchar_get_helper(keyPrefix + i, valPrefix + i); + test_varchar_get_helper(obTableClient, keyPrefix + i, valPrefix + i); } } finally { for (int i = 0; i < count; i++) { - client.delete("test_varchar_table", keyPrefix + i); + long affectedRows = obTableClient.delete("test_varchar_table", keyPrefix + i); + assertEquals(1, affectedRows); } } } - public void test_varchar_helper_thread(final String prefix, final int count) throws Exception { + public void test_varchar_helper_thread(final ObTableClient obTableClient, final String prefix, + final int count) throws Exception { new Thread(new Runnable() { @Override public void run() { try { long start = System.currentTimeMillis(); - test_varchar_helper(prefix, count); + test_varchar_helper(obTableClient, prefix, count); System.err.println("cost: " + (System.currentTimeMillis() - start)); } catch (Exception e) { e.printStackTrace(); @@ -994,18 +1307,22 @@ public void run() { } public void syncRefreshMetaHelper(final ObTableClient obTableClient) { - new Thread(new Runnable() { - @Override - public void run() { - for (int i = 0; i < 10; i++) { - try { - obTableClient.syncRefreshMetadata(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); + if (obTableClient.isOdpMode()) { + // do noting + } else { + new Thread(new Runnable() { + @Override + public void run() { + for (int i = 0; i < 10; i++) { + try { + obTableClient.syncRefreshMetadata(); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(); + } } } - } - }).start(); + }).start(); + } } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/bolt/ObTableTest.java b/src/test/java/com/alipay/oceanbase/rpc/bolt/ObTableTest.java index 8430d504..1526b6e9 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/bolt/ObTableTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/bolt/ObTableTest.java @@ -22,21 +22,36 @@ import com.alipay.oceanbase.rpc.table.ObTable; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class ObTableTest extends ObTableClientTestBase { private ObTable obTable; + @BeforeClass + static public void beforeTest() throws Exception { + ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + + assertFalse(obTableClient.isOdpMode()); + } + @Before public void setup() throws Exception { ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); obTableClient.init(); - obTable = obTableClient.getTable("test_varchar_table", new Object[] { "abc" }, true, true) - .getRight(); - client = obTable; + + if (obTableClient.isOdpMode()) { + obTableClient.close(); + throw new ObTableException("ODP Mode does not support this test"); + } else { + obTable = obTableClient + .getTable("test_varchar_table", new Object[] { "abc" }, true, true).getRight() + .getObTable(); + client = obTable; + } } @Test diff --git a/src/test/java/com/alipay/oceanbase/rpc/containerBase/ContainerTestBase.java b/src/test/java/com/alipay/oceanbase/rpc/containerBase/ContainerTestBase.java new file mode 100644 index 00000000..4a6a8cd6 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/containerBase/ContainerTestBase.java @@ -0,0 +1,96 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2023 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alipay.oceanbase.rpc.containerBase; + +import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; + +import java.time.Duration; +import java.util.Arrays; + +public class ContainerTestBase { + + private static final Logger logger = LoggerFactory + .getLogger(ContainerTestBase.class); + + protected static final Network NETWORK = Network.newNetwork(); + protected static final String CLUSTER_NAME = "obkv-table-client-java"; + protected static final String SYS_PASSWORD = "OB_ROOT_PASSWORD"; + protected static final String TEST_USERNAME = "root@test#" + CLUSTER_NAME; + + @ClassRule + public static final GenericContainer OB_SERVER = createOceanBaseCEContainer(); + + @SuppressWarnings({ "unchecked", "rawtypes" }) + protected static GenericContainer createOceanBaseCEContainer() { + GenericContainer container = new GenericContainer("oceanbase/oceanbase-ce:4.2.1_bp2") + .withNetwork(NETWORK) + .withExposedPorts(2881, 2882, 8080) + .withEnv("MODE", "slim") + .withEnv("OB_CLUSTER_NAME", CLUSTER_NAME) + .withEnv("OB_ROOT_PASSWORD", SYS_PASSWORD) + .withCopyFileToContainer(MountableFile.forClasspathResource("ci.sql"), + "/root/boot/init.d/init.sql") + .waitingFor( + Wait.forLogMessage(".*boot success!.*", 1) + .withStartupTimeout(Duration.ofMinutes(5))) + .withLogConsumer(new Slf4jLogConsumer(logger)); + container.setPortBindings(Arrays.asList("2881:2881", "2882:2882", "8080:8080")); + return container; + } + + @BeforeClass + public static void before() { + if (!ObTableClientTestUtil.FULL_USER_NAME.equals("full-user-name")) { + return; + } + + // Set config + ObTableClientTestUtil.PARAM_URL = "http://127.0.0.1:8080/services?Action=ObRootServiceInfo&ObCluster=" + + CLUSTER_NAME + "&database=test"; + ObTableClientTestUtil.FULL_USER_NAME = TEST_USERNAME; + ObTableClientTestUtil.PASSWORD = ""; + ObTableClientTestUtil.PROXY_SYS_USER_NAME = "root"; + ObTableClientTestUtil.PROXY_SYS_USER_PASSWORD = SYS_PASSWORD; + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/exception/ExceptionUtilTest.java b/src/test/java/com/alipay/oceanbase/rpc/exception/ExceptionUtilTest.java index 6e951c47..c21e1943 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/exception/ExceptionUtilTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/exception/ExceptionUtilTest.java @@ -30,15 +30,15 @@ public class ExceptionUtilTest { @Test public void testThrowException() { try { - ExceptionUtil.throwObTableException("0.0.0.0", 0, 0L, 0L, -4001); + ExceptionUtil.throwObTableException("0.0.0.0", 0, 0L, 0L, -4001, ""); } catch (ObTableException e) { Assert.assertTrue(e.getMessage().contains("OB_OBJ_TYPE_ERROR")); } try { - ExceptionUtil.throwObTableException("0.0.0.0", 0, 0L, 0L, -9013); + ExceptionUtil.throwObTableException("0.0.0.0", 0, 0L, 0L, -9013, ""); } catch (ObTableException e) { - Assert.assertTrue(e instanceof ObTableUnexpectedException); + Assert.assertTrue(e instanceof ObTableException); Assert.assertTrue(e.getMessage().contains("OB_OSS_WRITE_ERROR")); } @@ -57,7 +57,8 @@ public void testThrowException() { // convertToObTableException try { - ExceptionUtil.convertToObTableException("0.0.0.0", 0, 0L, 0L, OB_OSS_WRITE_ERROR); + ExceptionUtil.convertToObTableException("0.0.0.0", 0, 0L, 0L, + OB_OSS_WRITE_ERROR.errorCode, ""); } catch (ObTableException e) { Assert.assertTrue(e instanceof ObTableUnexpectedException); Assert.assertTrue(e.getMessage().contains("OB_OSS_WRITE_ERROR")); @@ -73,7 +74,7 @@ public void testThrowException() { }; for (ResultCodes c : resultCodes) { try { - ExceptionUtil.throwObTableException("0.0.0.0", 0, 0L, 0L, c.errorCode); + ExceptionUtil.throwObTableException("0.0.0.0", 0, 0L, 0L, c.errorCode, ""); } catch (ObTableException e) { Assert.assertFalse(e instanceof ObTableUnexpectedException); Assert.assertTrue(e instanceof ObTableException); diff --git a/src/test/java/com/alipay/oceanbase/rpc/hbase/ObHTableOperationRequest.java b/src/test/java/com/alipay/oceanbase/rpc/hbase/ObHTableOperationRequest.java index 6bb08090..7a24604d 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/hbase/ObHTableOperationRequest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/hbase/ObHTableOperationRequest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.rpc.hbase; +import com.alipay.oceanbase.rpc.protocol.payload.Constants; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableAbstractOperationRequest; @@ -123,7 +124,7 @@ public ObTableAbstractOperationRequest obTableOperationRequest() { ObTableOperationType.INSERT, new Object[] { rowKey, qualifierName, timestamp }, V_COLUMNS, new Object[] { value }, 10 * 1000); break; - case DEL: // TODO 如何 DEL？ + case DEL: request = ObTableOperationRequest.getInstance(getTargetTableName(), ObTableOperationType.DEL, new Object[] { rowKey, qualifierName, timestamp }, null, null, 10 * 1000); @@ -148,7 +149,70 @@ public ObTableAbstractOperationRequest obTableOperationRequest() { request.setTableName(getTargetTableName()); ((ObTableQueryRequest) request).setTableQuery(obTableQuery); - ((ObTableQueryRequest) request).setPartitionId(0); + ((ObTableQueryRequest) request).setPartitionId(Constants.INVALID_TABLET_ID); + ((ObTableQueryRequest) request).setTableId(Constants.OB_INVALID_ID); + break; + default: + throw new RuntimeException("operationType invalid: " + operationType); + } + + request.setEntityType(ObTableEntityType.HKV); + + return request; + + } + + public ObTableAbstractOperationRequest obTableGroupOperationRequest() { + ObTableAbstractOperationRequest request = null; + if (timestamp == -1) { + timestamp = System.currentTimeMillis(); + } + ObTableQuery obTableQuery = null; + ObHTableFilter filter = null; + ObNewRange obNewRange = null; + switch (operationType) { + case GET: + request = new ObTableQueryRequest(); + obTableQuery = new ObTableQuery(); + obTableQuery.setIndexName("PRIMARY"); + filter = new ObHTableFilter(); + obTableQuery.sethTableFilter(filter); + + obNewRange = new ObNewRange(); + obNewRange + .setStartKey(ObRowKey.getInstance(rowKey, ObObj.getMin(), ObObj.getMin())); + obNewRange.setEndKey(ObRowKey.getInstance(rowKey, ObObj.getMax(), ObObj.getMax())); + obTableQuery.addKeyRange(obNewRange); + obTableQuery.addSelectColumn("K"); + obTableQuery.addSelectColumn("Q"); + obTableQuery.addSelectColumn("T"); + obTableQuery.addSelectColumn("V"); + + request.setTableName(getTableName()); + ((ObTableQueryRequest) request).setTableQuery(obTableQuery); + ((ObTableQueryRequest) request).setPartitionId(Constants.INVALID_TABLET_ID); + + break; + case SCAN: + request = new ObTableQueryRequest(); + obTableQuery = new ObTableQuery(); + obTableQuery.setIndexName("PRIMARY"); + filter = new ObHTableFilter(); + obTableQuery.sethTableFilter(filter); + + obNewRange = new ObNewRange(); + obNewRange + .setStartKey(ObRowKey.getInstance(rowKey, ObObj.getMin(), ObObj.getMin())); + obNewRange.setEndKey(ObRowKey.getInstance(rowKey, ObObj.getMax(), ObObj.getMax())); + obTableQuery.addKeyRange(obNewRange); + obTableQuery.addSelectColumn("K"); + obTableQuery.addSelectColumn("Q"); + obTableQuery.addSelectColumn("T"); + obTableQuery.addSelectColumn("V"); + + request.setTableName(getTableName()); + ((ObTableQueryRequest) request).setTableQuery(obTableQuery); + ((ObTableQueryRequest) request).setPartitionId(Constants.INVALID_TABLET_ID); break; default: diff --git a/src/test/java/com/alipay/oceanbase/rpc/hbase/ObHTableTest.java b/src/test/java/com/alipay/oceanbase/rpc/hbase/ObHTableTest.java index 7988c0b7..f0faed35 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/hbase/ObHTableTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/hbase/ObHTableTest.java @@ -18,21 +18,47 @@ package com.alipay.oceanbase.rpc.hbase; import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.ObTableOperationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.ObTableQueryRequest; +import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObTable; import com.alipay.oceanbase.rpc.util.ObTableClientTestUtil; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; +import java.util.List; + +import static org.junit.Assert.assertFalse; + public class ObHTableTest { - private ObTable client; + private ObTable client; + private ObTableClient obTableClient; + + @BeforeClass + static public void beforeTest() throws Exception { + ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.init(); + obTableClient.setRunningMode(ObTableClient.RunningMode.HBASE); + + assertFalse(obTableClient.isOdpMode()); + } @Before public void setup() throws Exception { ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); obTableClient.init(); - client = obTableClient.getTable("test_varchar_table", new Object[] { "abc" }, true, true) - .getRight(); + if (obTableClient.isOdpMode()) { + obTableClient.close(); + throw new ObTableException("ODP Mode does not support this test"); + } else { + client = obTableClient + .getTable("test_varchar_table", new Object[] { "abc" }, true, true).getRight() + .getObTable(); + this.obTableClient = obTableClient; + } } @Test @@ -58,18 +84,146 @@ PRIMARY KEY (`K`, `Q`, `T`) hTableOperationRequest.obTableOperationRequest(), 1000000); System.err.println(obj); - hTableOperationRequest = new ObHTableOperationRequest(); - hTableOperationRequest.setOperationType(ObTableOperationType.GET); - hTableOperationRequest.setTableName("test_hbase"); - hTableOperationRequest.setFamilyName("fn"); - hTableOperationRequest.setRowKey("key".getBytes()); - hTableOperationRequest.setQualifierName("qualifierName".getBytes()); - hTableOperationRequest.setValue("value".getBytes()); - client.getRealClient().invokeSync(client.getConnection(), - hTableOperationRequest.obTableOperationRequest(), 1000000); - client.delete("test_hbase$fn", new Object[] { "key".getBytes(), "qualifierName1".getBytes(), 12323121L }); } + public byte[][] extractFamilyFromQualifier(byte[] qualifier) throws Exception { + int total_length = qualifier.length; + int familyLen = -1; + byte[][] familyAndQualifier = new byte[2][]; + + for (int i = 0; i < total_length; i++) { + if (qualifier[i] == '\0') { + familyLen = i; + break; + } + } + + byte[] family = new byte[familyLen]; + if (familyLen != -1) { + for (int i = 0; i < familyLen; i++) { + family[i] = qualifier[i]; + } + } else { + throw new RuntimeException("can not get family name"); + } + familyAndQualifier[0] = family; + int qualifierLen = total_length - familyLen - 1; + byte[] newQualifier = new byte[qualifierLen]; + if (qualifierLen > 0) { + for (int i = 0; i < qualifierLen; i++) { + newQualifier[i] = qualifier[i + familyLen + 1]; + } + } else { + throw new RuntimeException("can not get qualifier name"); + } + for (int i = 0; i < qualifierLen; i++) { + newQualifier[i] = qualifier[i + familyLen + 1]; + } + familyAndQualifier[1] = newQualifier; + System.out.println(newQualifier); + System.out.println(family); + return familyAndQualifier; + } + + public void getKeyValueFromResult(ObTableClientQueryStreamResult clientQueryStreamResult, + boolean isTableGroup, byte[] family) throws Exception { + for (List row : clientQueryStreamResult.getCacheRows()) { + System.out.println(new String((byte[]) row.get(0).getValue())); //K + // System.out.println(family); //family + if (isTableGroup) { + byte[][] familyAndQualifier = extractFamilyFromQualifier((byte[]) row.get(1) + .getValue()); + System.out.println(new String(familyAndQualifier[0])); //family + System.out.println(new String(familyAndQualifier[1])); //qualifier + } else { + System.out.println(family); //family + System.out.println(new String((byte[]) row.get(1).getValue())); //qualifier + } + + System.out.println((Long) row.get(2).getValue());//T + System.out.println(new String((byte[]) row.get(3).getValue()));//V + } + } + + @Test + public void hbaseTableGroupTest() throws Exception { + /* + CREATE TABLEGROUP test SHARDING = 'ADAPTIVE'; + CREATE TABLE `test$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = test; + */ + byte[] family = new byte[] {}; + ObHTableOperationRequest hTableOperationRequestGet = new ObHTableOperationRequest(); + hTableOperationRequestGet.setOperationType(ObTableOperationType.GET); + hTableOperationRequestGet.setTableName("test"); + hTableOperationRequestGet.setRowKey("putKey".getBytes()); + + ObTableQueryRequest requestGet = (ObTableQueryRequest) hTableOperationRequestGet + .obTableGroupOperationRequest(); + ObTableClientQueryStreamResult clientQueryStreamResultGet = (ObTableClientQueryStreamResult) obTableClient + .execute(requestGet); + + // Thread.currentThread().sleep(30000); + ObHTableOperationRequest hTableOperationRequestScan = new ObHTableOperationRequest(); + hTableOperationRequestScan.setOperationType(ObTableOperationType.SCAN); + hTableOperationRequestScan.setTableName("test"); + hTableOperationRequestScan.setRowKey("putKey".getBytes()); + + ObTableQueryRequest requestScan = (ObTableQueryRequest) hTableOperationRequestScan + .obTableGroupOperationRequest(); + ObTableClientQueryStreamResult clientQueryStreamResultScan = (ObTableClientQueryStreamResult) obTableClient + .execute(requestScan); + + } + + @Test + public void hbaseDiffTableGroupTest() throws Exception { + /* + CREATE TABLEGROUP test SHARDING = 'ADAPTIVE'; + CREATE TABLE `test$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = test; + CREATE TABLEGROUP test2 SHARDING = 'ADAPTIVE'; + CREATE TABLE `test2$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = test2; + */ + byte[] family = new byte[] {}; + ObHTableOperationRequest hTableOperationRequestGet = new ObHTableOperationRequest(); + hTableOperationRequestGet.setOperationType(ObTableOperationType.GET); + hTableOperationRequestGet.setTableName("test"); + hTableOperationRequestGet.setRowKey("putKey".getBytes()); + + ObTableQueryRequest requestGet = (ObTableQueryRequest) hTableOperationRequestGet + .obTableGroupOperationRequest(); + ObTableClientQueryStreamResult clientQueryStreamResultGet = (ObTableClientQueryStreamResult) obTableClient + .execute(requestGet); + + // Thread.currentThread().sleep(30000); + ObHTableOperationRequest hTableOperationRequestScan = new ObHTableOperationRequest(); + hTableOperationRequestScan.setOperationType(ObTableOperationType.SCAN); + hTableOperationRequestScan.setTableName("test2"); + hTableOperationRequestScan.setRowKey("putKey".getBytes()); + + ObTableQueryRequest requestScan = (ObTableQueryRequest) hTableOperationRequestScan + .obTableGroupOperationRequest(); + ObTableClientQueryStreamResult clientQueryStreamResultScan = (ObTableClientQueryStreamResult) obTableClient + .execute(requestScan); + } + } diff --git a/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObHashPartDescTest.java b/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObHashPartDescTest.java index 58541158..424d5130 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObHashPartDescTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObHashPartDescTest.java @@ -23,11 +23,13 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.ObColumn; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; import com.alipay.oceanbase.rpc.protocol.payload.impl.column.ObSimpleColumn; +import com.alipay.oceanbase.rpc.mutation.*; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -61,204 +63,586 @@ public void setUp() { @Test public void testGetPartId() { - - Assert.assertEquals(0, - (long) obHashPartDesc.getPartId(0, "column_1", System.currentTimeMillis())); - Assert.assertEquals(1, - (long) obHashPartDesc.getPartId(1, "column_1", System.currentTimeMillis())); - Assert.assertEquals(2, - (long) obHashPartDesc.getPartId(2, "column_1", System.currentTimeMillis())); - Assert.assertEquals(3, - (long) obHashPartDesc.getPartId(3, "column_1", System.currentTimeMillis())); - Assert.assertEquals(4, - (long) obHashPartDesc.getPartId(4, "column_1", System.currentTimeMillis())); - Assert.assertEquals(5, - (long) obHashPartDesc.getPartId(5, "column_1", System.currentTimeMillis())); - Assert.assertEquals(6, - (long) obHashPartDesc.getPartId(6, "column_1", System.currentTimeMillis())); - Assert.assertEquals(7, - (long) obHashPartDesc.getPartId(7, "column_1", System.currentTimeMillis())); - Assert.assertEquals(8, - (long) obHashPartDesc.getPartId(8, "column_1", System.currentTimeMillis())); - Assert.assertEquals(9, - (long) obHashPartDesc.getPartId(9, "column_1", System.currentTimeMillis())); - Assert.assertEquals(10, - (long) obHashPartDesc.getPartId(10, "column_1", System.currentTimeMillis())); - Assert.assertEquals(11, - (long) obHashPartDesc.getPartId(11, "column_1", System.currentTimeMillis())); - Assert.assertEquals(12, - (long) obHashPartDesc.getPartId(12, "column_1", System.currentTimeMillis())); - Assert.assertEquals(13, - (long) obHashPartDesc.getPartId(13, "column_1", System.currentTimeMillis())); - Assert.assertEquals(14, - (long) obHashPartDesc.getPartId(14, "column_1", System.currentTimeMillis())); - Assert.assertEquals(15, - (long) obHashPartDesc.getPartId(15, "column_1", System.currentTimeMillis())); - - Assert.assertEquals(0, - (long) obHashPartDesc.getPartId(-0, "column_1", System.currentTimeMillis())); - Assert.assertEquals(1, - (long) obHashPartDesc.getPartId(-1, "column_1", System.currentTimeMillis())); - Assert.assertEquals(2, - (long) obHashPartDesc.getPartId(-2, "column_1", System.currentTimeMillis())); - Assert.assertEquals(3, - (long) obHashPartDesc.getPartId(-3, "column_1", System.currentTimeMillis())); - Assert.assertEquals(4, - (long) obHashPartDesc.getPartId(-4, "column_1", System.currentTimeMillis())); - Assert.assertEquals(5, - (long) obHashPartDesc.getPartId(-5, "column_1", System.currentTimeMillis())); - Assert.assertEquals(6, - (long) obHashPartDesc.getPartId(-6, "column_1", System.currentTimeMillis())); - Assert.assertEquals(7, - (long) obHashPartDesc.getPartId(-7, "column_1", System.currentTimeMillis())); - Assert.assertEquals(8, - (long) obHashPartDesc.getPartId(-8, "column_1", System.currentTimeMillis())); - Assert.assertEquals(9, - (long) obHashPartDesc.getPartId(-9, "column_1", System.currentTimeMillis())); - Assert.assertEquals(10, - (long) obHashPartDesc.getPartId(-10, "column_1", System.currentTimeMillis())); - Assert.assertEquals(11, - (long) obHashPartDesc.getPartId(-11, "column_1", System.currentTimeMillis())); - Assert.assertEquals(12, - (long) obHashPartDesc.getPartId(-12, "column_1", System.currentTimeMillis())); - Assert.assertEquals(13, - (long) obHashPartDesc.getPartId(-13, "column_1", System.currentTimeMillis())); - Assert.assertEquals(14, - (long) obHashPartDesc.getPartId(-14, "column_1", System.currentTimeMillis())); - Assert.assertEquals(15, - (long) obHashPartDesc.getPartId(-15, "column_1", System.currentTimeMillis())); - - Assert.assertEquals(obHashPartDesc.getPartId(1, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("1", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(2, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("2", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(3, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("3", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(4, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("4", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(5, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("5", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(6, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("6", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(7, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("7", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(8, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("8", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(9, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("9", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(10, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("10", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(11, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("11", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(12, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("12", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(13, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("13", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(14, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("14", "column_1", System.currentTimeMillis())); - Assert.assertEquals(obHashPartDesc.getPartId(15, "column_1", System.currentTimeMillis()), - obHashPartDesc.getPartId("15", "column_1", System.currentTimeMillis())); - + // set values + Map values0 = new HashMap() { + { + put("K", 0); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values1 = new HashMap() { + { + put("K", 1); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values2 = new HashMap() { + { + put("K", 2); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values3 = new HashMap() { + { + put("K", 3); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values4 = new HashMap() { + { + put("K", 4); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values5 = new HashMap() { + { + put("K", 5); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values6 = new HashMap() { + { + put("K", 6); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values7 = new HashMap() { + { + put("K", 7); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values8 = new HashMap() { + { + put("K", 8); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values9 = new HashMap() { + { + put("K", 9); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values10 = new HashMap() { + { + put("K", 10); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values11 = new HashMap() { + { + put("K", 11); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values12 = new HashMap() { + { + put("K", 12); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values13 = new HashMap() { + { + put("K", 13); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values14 = new HashMap() { + { + put("K", 14); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values15 = new HashMap() { + { + put("K", 15); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + + Map values_0 = new HashMap() { + { + put("K", -0); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_1 = new HashMap() { + { + put("K", -1); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_2 = new HashMap() { + { + put("K", -2); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_3 = new HashMap() { + { + put("K", -3); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_4 = new HashMap() { + { + put("K", -4); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_5 = new HashMap() { + { + put("K", -5); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_6 = new HashMap() { + { + put("K", -6); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_7 = new HashMap() { + { + put("K", -7); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_8 = new HashMap() { + { + put("K", -8); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_9 = new HashMap() { + { + put("K", -9); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_10 = new HashMap() { + { + put("K", -10); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_11 = new HashMap() { + { + put("K", -11); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_12 = new HashMap() { + { + put("K", -12); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_13 = new HashMap() { + { + put("K", -13); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_14 = new HashMap() { + { + put("K", -14); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_15 = new HashMap() { + { + put("K", -15); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + + Map values0_e = new HashMap() { + { + put("K", 0); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values0_l = new HashMap() { + { + put("K", 0); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values1_e = new HashMap() { + { + put("K", 1); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values1_l = new HashMap() { + { + put("K", 1); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values2_e = new HashMap() { + { + put("K", 2); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values2_l = new HashMap() { + { + put("K", 2); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values3_e = new HashMap() { + { + put("K", 3); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values3_l = new HashMap() { + { + put("K", 3); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values4_e = new HashMap() { + { + put("K", 4); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values4_l = new HashMap() { + { + put("K", 4); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values5_e = new HashMap() { + { + put("K", 5); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values5_l = new HashMap() { + { + put("K", 5); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values6_e = new HashMap() { + { + put("K", 6); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values6_l = new HashMap() { + { + put("K", 6); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values7_e = new HashMap() { + { + put("K", 7); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values7_l = new HashMap() { + { + put("K", 7); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values8_e = new HashMap() { + { + put("K", 8); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values8_l = new HashMap() { + { + put("K", 8); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values9_e = new HashMap() { + { + put("K", 9); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values9_l = new HashMap() { + { + put("K", 9); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values10_e = new HashMap() { + { + put("K", 10); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values10_l = new HashMap() { + { + put("K", 10); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + + // test getPartId interface + Assert.assertEquals(0, (long) obHashPartDesc.getPartId(new Row(values0))); + Assert.assertEquals(1, (long) obHashPartDesc.getPartId(new Row(values1))); + Assert.assertEquals(2, (long) obHashPartDesc.getPartId(new Row(values2))); + Assert.assertEquals(3, (long) obHashPartDesc.getPartId(new Row(values3))); + Assert.assertEquals(4, (long) obHashPartDesc.getPartId(new Row(values4))); + Assert.assertEquals(5, (long) obHashPartDesc.getPartId(new Row(values5))); + Assert.assertEquals(6, (long) obHashPartDesc.getPartId(new Row(values6))); + Assert.assertEquals(7, (long) obHashPartDesc.getPartId(new Row(values7))); + Assert.assertEquals(8, (long) obHashPartDesc.getPartId(new Row(values8))); + Assert.assertEquals(9, (long) obHashPartDesc.getPartId(new Row(values9))); + Assert.assertEquals(10, (long) obHashPartDesc.getPartId(new Row(values10))); + Assert.assertEquals(11, (long) obHashPartDesc.getPartId(new Row(values11))); + Assert.assertEquals(12, (long) obHashPartDesc.getPartId(new Row(values12))); + Assert.assertEquals(13, (long) obHashPartDesc.getPartId(new Row(values13))); + Assert.assertEquals(14, (long) obHashPartDesc.getPartId(new Row(values14))); + Assert.assertEquals(15, (long) obHashPartDesc.getPartId(new Row(values15))); + + Assert.assertEquals(0, (long) obHashPartDesc.getPartId(new Row(values_0))); + Assert.assertEquals(1, (long) obHashPartDesc.getPartId(new Row(values_1))); + Assert.assertEquals(2, (long) obHashPartDesc.getPartId(new Row(values_2))); + Assert.assertEquals(3, (long) obHashPartDesc.getPartId(new Row(values_3))); + Assert.assertEquals(4, (long) obHashPartDesc.getPartId(new Row(values_4))); + Assert.assertEquals(5, (long) obHashPartDesc.getPartId(new Row(values_5))); + Assert.assertEquals(6, (long) obHashPartDesc.getPartId(new Row(values_6))); + Assert.assertEquals(7, (long) obHashPartDesc.getPartId(new Row(values_7))); + Assert.assertEquals(8, (long) obHashPartDesc.getPartId(new Row(values_8))); + Assert.assertEquals(9, (long) obHashPartDesc.getPartId(new Row(values_9))); + Assert.assertEquals(10, (long) obHashPartDesc.getPartId(new Row(values_10))); + Assert.assertEquals(11, (long) obHashPartDesc.getPartId(new Row(values_11))); + Assert.assertEquals(12, (long) obHashPartDesc.getPartId(new Row(values_12))); + Assert.assertEquals(13, (long) obHashPartDesc.getPartId(new Row(values_13))); + Assert.assertEquals(14, (long) obHashPartDesc.getPartId(new Row(values_14))); + Assert.assertEquals(15, (long) obHashPartDesc.getPartId(new Row(values_15))); + + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values0_e)), + obHashPartDesc.getPartId(new Row(values0_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values1_e)), + obHashPartDesc.getPartId(new Row(values1_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values2_e)), + obHashPartDesc.getPartId(new Row(values2_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values3_e)), + obHashPartDesc.getPartId(new Row(values3_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values4_e)), + obHashPartDesc.getPartId(new Row(values4_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values5_e)), + obHashPartDesc.getPartId(new Row(values5_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values6_e)), + obHashPartDesc.getPartId(new Row(values6_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values7_e)), + obHashPartDesc.getPartId(new Row(values7_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values8_e)), + obHashPartDesc.getPartId(new Row(values8_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values9_e)), + obHashPartDesc.getPartId(new Row(values9_l))); + Assert.assertEquals(obHashPartDesc.getPartId(new Row(values10_e)), + obHashPartDesc.getPartId(new Row(values10_l))); } @Test public void testGetPartIds() { - Object[] rowKey_0 = new Object[] { 0, "column_1", System.currentTimeMillis() }; - - Object[] rowKey_8 = new Object[] { 8, "column_1", System.currentTimeMillis() }; - - Object[] rowKey_15 = new Object[] { 15, "column_1", System.currentTimeMillis() }; - - Object[] rowKey_16 = new Object[] { 16, "column_1", System.currentTimeMillis() }; - - Object[] rowKey_30 = new Object[] { 30, "column_1", System.currentTimeMillis() }; - - Object[] rowKey_8f = new Object[] { -8, "column_1", System.currentTimeMillis() }; - - Object[] rowKey_15f = new Object[] { -15, "column_1", System.currentTimeMillis() }; - - Object[] rowKey_30f = new Object[] { -30, "column_1", System.currentTimeMillis() }; + Map values0 = new HashMap() { + { + put("K", 0); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values8 = new HashMap() { + { + put("K", 8); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values15 = new HashMap() { + { + put("K", 15); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values16 = new HashMap() { + { + put("K", 16); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values30 = new HashMap() { + { + put("K", 30); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_8f = new HashMap() { + { + put("K", -8); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_15f = new HashMap() { + { + put("K", -15); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map values_30f = new HashMap() { + { + put("K", -30); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; Assert.assertEquals(buildPartIds(0, 0), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values0), true)); Assert.assertEquals(buildEmptyPartIds(), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values0), false)); Assert.assertEquals(buildEmptyPartIds(), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values0), true)); Assert.assertEquals(buildEmptyPartIds(), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values0), false)); Assert.assertEquals(buildEmptyPartIds(), - obHashPartDesc.getPartIds(rowKey_8, true, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values8), true, new Row(values0), true)); Assert.assertEquals(buildEmptyPartIds(), - obHashPartDesc.getPartIds(rowKey_8, false, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values8), false, new Row(values0), true)); Assert.assertEquals(buildEmptyPartIds(), - obHashPartDesc.getPartIds(rowKey_8, true, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values8), true, new Row(values0), false)); Assert.assertEquals(buildEmptyPartIds(), - obHashPartDesc.getPartIds(rowKey_8, false, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values8), false, new Row(values0), false)); Assert.assertEquals(buildPartIds(0, 8), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_8, true)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values8), true)); Assert.assertEquals(buildPartIds(1, 8), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_8, true)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values8), true)); Assert.assertEquals(buildPartIds(0, 7), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_8, false)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values8), false)); Assert.assertEquals(buildPartIds(1, 7), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_8, false)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values8), false)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_15, true)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values15), true)); Assert.assertEquals(buildPartIds(0, 14), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_15, false)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values15), false)); Assert.assertEquals(buildPartIds(1, 15), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_15, true)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values15), true)); Assert.assertEquals(buildPartIds(1, 14), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_15, false)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values15), false)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_16, true)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values16), true)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_16, false)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values16), false)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_16, true)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values16), true)); Assert.assertEquals(buildPartIds(1, 15), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_16, false)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values16), false)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_30, true)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values30), true)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_0, true, rowKey_30, false)); + obHashPartDesc.getPartIds(new Row(values0), true, new Row(values30), false)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_30, true)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values30), true)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_0, false, rowKey_30, false)); + obHashPartDesc.getPartIds(new Row(values0), false, new Row(values30), false)); Assert.assertEquals(buildPartIds(-8, 0), - obHashPartDesc.getPartIds(rowKey_8f, true, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values_8f), true, new Row(values0), true)); Assert.assertEquals(buildPartIds(-8, -1), - obHashPartDesc.getPartIds(rowKey_8f, true, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values_8f), true, new Row(values0), false)); Assert.assertEquals(buildPartIds(-7, 0), - obHashPartDesc.getPartIds(rowKey_8f, false, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values_8f), false, new Row(values0), true)); Assert.assertEquals(buildPartIds(-7, -1), - obHashPartDesc.getPartIds(rowKey_8f, false, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values_8f), false, new Row(values0), false)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_15f, true, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values_15f), true, new Row(values0), true)); Assert.assertEquals(buildPartIds(-15, -1), - obHashPartDesc.getPartIds(rowKey_15f, true, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values_15f), true, new Row(values0), false)); Assert.assertEquals(buildPartIds(-14, 0), - obHashPartDesc.getPartIds(rowKey_15f, false, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values_15f), false, new Row(values0), true)); Assert.assertEquals(buildPartIds(-14, -1), - obHashPartDesc.getPartIds(rowKey_15f, false, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values_15f), false, new Row(values0), false)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_30f, true, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values_30f), true, new Row(values0), true)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_30f, true, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values_30f), true, new Row(values0), false)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_30f, false, rowKey_0, true)); + obHashPartDesc.getPartIds(new Row(values_30f), false, new Row(values0), true)); Assert.assertEquals(buildPartIds(0, 15), - obHashPartDesc.getPartIds(rowKey_30f, false, rowKey_0, false)); + obHashPartDesc.getPartIds(new Row(values_30f), false, new Row(values0), false)); } private List buildPartIds(long start, long end) { diff --git a/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDescTest.java b/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDescTest.java index eb754c2b..ba5a6c3c 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDescTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDescTest.java @@ -19,6 +19,7 @@ import com.alipay.oceanbase.rpc.location.LocationUtil; import com.alipay.oceanbase.rpc.location.model.TableEntry; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObCollationType; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObColumn; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; @@ -28,9 +29,7 @@ import org.junit.Before; import org.junit.Test; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.alipay.oceanbase.rpc.location.model.partition.ObPartFuncType.KEY_V3; @@ -100,171 +99,338 @@ public void setUp() { partColumns.add(column); keyUtf8.setPartColumns(partColumns); keyUtf8.setRowKeyElement(TableEntry.HBASE_ROW_KEY_ELEMENT); - keyUtf8.prepare(); + // keyUtf8.prepare(); } @Test public void testGetPartId() { // key binary - long partId = keyBinary.getPartId("partition_1", "column_1", System.currentTimeMillis()); - Assert.assertEquals(2, partId); - - Assert.assertEquals( - keyBinary.getPartId("partition_1", "column_1", System.currentTimeMillis()), - keyBinary.getPartId("partition_2", "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - keyBinary.getPartId("partition_1", "column_1", System.currentTimeMillis()), - keyBinary.getPartId("partition_1".getBytes(), "column_1", System.currentTimeMillis())); - - Assert.assertEquals(keyBinary.getPartId("test_1", "column_1", System.currentTimeMillis()), - keyBinary.getPartId("test_2", "column_1", System.currentTimeMillis())); - - Assert.assertEquals(keyBinary.getPartId("test_1", "column_1", System.currentTimeMillis()), - keyBinary.getPartId("test_2".getBytes(), "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - keyBinary.getPartId("test_1".getBytes(), "column_1", System.currentTimeMillis()), - keyBinary.getPartId("test_2".getBytes(), "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - keyUtf8_CI.getPartId("partition_1", "column_1", System.currentTimeMillis()), - keyUtf8_CI.getPartId("Partition_1".getBytes(), "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - keyUtf8_CI.getPartId("partition_1", "column_1", System.currentTimeMillis()), - keyUtf8_CI.getPartId("Partition_2".getBytes(), "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - keyUtf8.getPartId("partition_1", "column_1", System.currentTimeMillis()), - keyUtf8.getPartId("partition_2", "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - keyUtf8.getPartId("partition_1", "column_1", System.currentTimeMillis()), - keyUtf8.getPartId("partition_2".getBytes(), "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - keyUtf8.getPartId("partition_1", "column_1", System.currentTimeMillis()), - keyUtf8.getPartId("partition_2", "column_1", System.currentTimeMillis())); - - Assert.assertNotEquals( - keyUtf8.getPartId("partition_1", "column_1", System.currentTimeMillis()), - keyUtf8.getPartId("Partition_1", "column_1", System.currentTimeMillis())); - - Assert.assertNotEquals( - keyUtf8_CI.getPartId("PARTITION_1", "column_1", System.currentTimeMillis()), - keyUtf8.getPartId("partition_1".getBytes(), "column_1", System.currentTimeMillis())); - - Assert.assertNotEquals( - keyUtf8_CI.getPartId("partition_1", "column_1", System.currentTimeMillis()), - keyBinary.getPartId("PARTITION_1".getBytes(), "column_1", System.currentTimeMillis())); + Map partition_1_test = new HashMap() { + { + put("K", "partition_1"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + long partId = keyBinary.getPartId(new Row(partition_1_test)); + Assert.assertEquals(11, partId); + + Map partition_2_test = new HashMap() { + { + put("K", "partition_2"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Assert.assertEquals(keyBinary.getPartId(new Row(partition_1_test)), + keyBinary.getPartId(new Row(partition_2_test))); + + Map partition_1_bytes_test = new HashMap() { + { + put("K", "partition_1".getBytes()); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map partition_2_bytes_test = new HashMap() { + { + put("K", "partition_2".getBytes()); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Assert.assertEquals(keyBinary.getPartId(new Row(partition_1_test)), + keyBinary.getPartId(new Row(partition_1_bytes_test))); + + Map test_1 = new HashMap() { + { + put("K", "test_1"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map test_2 = new HashMap() { + { + put("K", "test_2"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map test_1_bytes = new HashMap() { + { + put("K", "test_1".getBytes()); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map test_2_bytes = new HashMap() { + { + put("K", "test_2".getBytes()); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Assert.assertEquals(keyBinary.getPartId(new Row(test_1)), + keyBinary.getPartId(new Row(test_2))); + Assert.assertEquals(keyBinary.getPartId(new Row(test_1)), + keyBinary.getPartId(new Row(test_2_bytes))); + + Assert.assertEquals(keyBinary.getPartId(new Row(test_1_bytes)), + keyBinary.getPartId(new Row(test_2_bytes))); + + Map Partition_1_test = new HashMap() { + { + put("K", "Partition_1"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map Partition_1_bytes_test = new HashMap() { + { + put("K", "Partition_1".getBytes()); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map Partition_2_bytes_test = new HashMap() { + { + put("K", "Partition_2".getBytes()); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map PARTITION_1_test = new HashMap() { + { + put("K", "PARTITION_1"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map PARTITION_1_bytes_test = new HashMap() { + { + put("K", "PARTITION_1".getBytes()); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Assert.assertEquals(keyUtf8_CI.getPartId(new Row(partition_1_test)), + keyUtf8_CI.getPartId(new Row(Partition_1_bytes_test))); + Assert.assertEquals(keyUtf8_CI.getPartId(new Row(partition_1_test)), + keyUtf8_CI.getPartId(new Row(Partition_2_bytes_test))); + Assert.assertEquals(keyUtf8.getPartId(new Row(partition_1_test)), + keyUtf8.getPartId(new Row(partition_2_test))); + Assert.assertEquals(keyUtf8.getPartId(new Row(partition_1_test)), + keyUtf8.getPartId(new Row(partition_2_bytes_test))); + + Assert.assertEquals(keyUtf8.getPartId(new Row(partition_1_test)), + keyUtf8.getPartId(new Row(partition_2_test))); + + Assert.assertNotEquals(keyUtf8.getPartId(new Row(partition_1_test)), + keyUtf8.getPartId(new Row(Partition_1_test))); + + Assert.assertNotEquals(keyUtf8_CI.getPartId(new Row(PARTITION_1_test)), + keyUtf8.getPartId(new Row(partition_1_bytes_test))); + + Assert.assertNotEquals(keyUtf8_CI.getPartId(new Row(partition_1_test)), + keyBinary.getPartId(new Row(PARTITION_1_bytes_test))); } @Test public void testGetPartIds() { long timestamp = System.currentTimeMillis(); - Object[] startKey1 = new Object[] { "partition_1", "column_1", timestamp }; - Object[] endKey1 = new Object[] { "partition_2", "column_1", timestamp }; + Map startKey1 = new HashMap() { + { + put("K", "partition_1"); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey1 = new HashMap() { + { + put("K", "partition_2"); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey2 = new HashMap() { + { + put("K", "partition_1".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey2 = new HashMap() { + { + put("K", "partition_2".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey3 = new HashMap() { + { + put("K", "test_1".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey3 = new HashMap() { + { + put("K", "test_2".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey4 = new HashMap() { + { + put("K", "PARTITION_1"); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey4 = new HashMap() { + { + put("K", "PARTITION_2"); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey5 = new HashMap() { + { + put("K", "PARTITION_1".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey5 = new HashMap() { + { + put("K", "PARTITION_2".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey6 = new HashMap() { + { + put("K", "TEST_1".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey6 = new HashMap() { + { + put("K", "TEST_2".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Assert.assertEquals(keyBinary.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + keyBinary.getPartIds(new Row(startKey2), true, new Row(endKey2), true)); + + Assert.assertEquals(keyBinary.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + keyBinary.getPartIds(new Row(startKey2), true, new Row(endKey1), true)); - Object[] startKey2 = new Object[] { "partition_1".getBytes(), "column_1", timestamp }; - Object[] endKey2 = new Object[] { "partition_2".getBytes(), "column_1", timestamp }; - - Object[] startKey3 = new Object[] { "test_1".getBytes(), "column_1", timestamp }; - Object[] endKey3 = new Object[] { "test_2".getBytes(), "column_1", timestamp }; - - Object[] startKey4 = new Object[] { "PARTITION_1", "column_1", timestamp }; - Object[] endKey4 = new Object[] { "PARTITION_2", "column_1", timestamp }; - - Object[] startKey5 = new Object[] { "PARTITION_1".getBytes(), "column_1", timestamp }; - Object[] endKey5 = new Object[] { "PARTITION_2".getBytes(), "column_1", timestamp }; - - Object[] startKey6 = new Object[] { "TEST_1".getBytes(), "column_1", timestamp }; - Object[] endKey6 = new Object[] { "TEST_2".getBytes(), "column_1", timestamp }; - - Assert.assertEquals(keyBinary.getPartIds(startKey1, true, endKey1, true), - keyBinary.getPartIds(startKey2, true, endKey2, true)); - - Assert.assertEquals(keyBinary.getPartIds(startKey1, true, endKey2, true), - keyBinary.getPartIds(startKey2, true, endKey1, true)); - - Assert.assertEquals(keyBinary.getPartIds(startKey1, false, endKey2, false), - keyBinary.getPartIds(startKey2, false, endKey1, false)); + Assert.assertEquals( + keyBinary.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + keyBinary.getPartIds(new Row(startKey2), false, new Row(endKey1), false)); try { - keyBinary.getPartIds(startKey1, false, endKey3, false); - Assert.fail(); + List ans = keyBinary.getPartIds(new Row(startKey1), false, new Row(endKey3), + false); + Assert.assertEquals(16, ans.size()); } catch (Exception e) { - + e.printStackTrace(); + Assert.assertTrue(false); } try { - keyBinary.getPartIds(startKey3, false, endKey1, true); - Assert.fail(); + List ans = keyBinary + .getPartIds(new Row(startKey3), false, new Row(endKey1), true); + Assert.assertEquals(16, ans.size()); } catch (Exception e) { - + e.printStackTrace(); + Assert.assertTrue(false); } try { - keyBinary.getPartIds(startKey1, false, endKey4, true); - Assert.fail(); + List ans = keyBinary + .getPartIds(new Row(startKey1), false, new Row(endKey4), true); + Assert.assertEquals(16, ans.size()); } catch (Exception e) { - + e.printStackTrace(); + Assert.assertTrue(false); } - Assert.assertEquals(keyUtf8_CI.getPartIds(startKey1, true, endKey1, true), - keyUtf8_CI.getPartIds(startKey4, true, endKey4, true)); + Assert.assertEquals( + keyUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + keyUtf8_CI.getPartIds(new Row(startKey4), true, new Row(endKey4), true)); - Assert.assertEquals(keyUtf8_CI.getPartIds(startKey2, true, endKey2, true), - keyUtf8_CI.getPartIds(startKey5, true, endKey5, true)); + Assert.assertEquals( + keyUtf8_CI.getPartIds(new Row(startKey2), true, new Row(endKey2), true), + keyUtf8_CI.getPartIds(new Row(startKey5), true, new Row(endKey5), true)); - Assert.assertEquals(keyUtf8_CI.getPartIds(startKey3, false, endKey3, false), - keyUtf8_CI.getPartIds(startKey6, false, endKey6, false)); + Assert.assertEquals( + keyUtf8_CI.getPartIds(new Row(startKey3), false, new Row(endKey3), false), + keyUtf8_CI.getPartIds(new Row(startKey6), false, new Row(endKey6), false)); - Assert.assertEquals(keyUtf8_CI.getPartIds(startKey1, true, endKey1, true), - keyUtf8_CI.getPartIds(startKey2, true, endKey2, true)); + Assert.assertEquals( + keyUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + keyUtf8_CI.getPartIds(new Row(startKey2), true, new Row(endKey2), true)); - Assert.assertEquals(keyUtf8_CI.getPartIds(startKey1, true, endKey2, true), - keyUtf8_CI.getPartIds(startKey2, true, endKey1, true)); + Assert.assertEquals( + keyUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + keyUtf8_CI.getPartIds(new Row(startKey2), true, new Row(endKey1), true)); - Assert.assertEquals(keyUtf8_CI.getPartIds(startKey1, false, endKey2, false), - keyUtf8_CI.getPartIds(startKey2, false, endKey1, false)); + Assert.assertEquals( + keyUtf8_CI.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + keyUtf8_CI.getPartIds(new Row(startKey2), false, new Row(endKey1), false)); - Assert.assertEquals(keyUtf8_CI.getPartIds(startKey1, true, endKey1, true), - keyUtf8_CI.getPartIds(startKey4, true, endKey4, true)); + Assert.assertEquals( + keyUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + keyUtf8_CI.getPartIds(new Row(startKey4), true, new Row(endKey4), true)); - Assert.assertEquals(keyUtf8_CI.getPartIds(startKey1, true, endKey2, true), - keyUtf8_CI.getPartIds(startKey5, true, endKey5, true)); + Assert.assertEquals( + keyUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + keyUtf8_CI.getPartIds(new Row(startKey5), true, new Row(endKey5), true)); - Assert.assertEquals(keyUtf8_CI.getPartIds(startKey1, false, endKey2, false), - keyUtf8_CI.getPartIds(startKey5, false, endKey4, false)); + Assert.assertEquals( + keyUtf8_CI.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + keyUtf8_CI.getPartIds(new Row(startKey5), false, new Row(endKey4), false)); - Assert.assertEquals(keyUtf8.getPartIds(startKey1, true, endKey1, true), - keyUtf8.getPartIds(startKey2, true, endKey2, true)); + Assert.assertEquals(keyUtf8.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + keyUtf8.getPartIds(new Row(startKey2), true, new Row(endKey2), true)); - Assert.assertEquals(keyUtf8.getPartIds(startKey1, true, endKey2, true), - keyUtf8.getPartIds(startKey2, true, endKey1, true)); + Assert.assertEquals(keyUtf8.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + keyUtf8.getPartIds(new Row(startKey2), true, new Row(endKey1), true)); - Assert.assertEquals(keyUtf8.getPartIds(startKey1, false, endKey2, false), - keyUtf8.getPartIds(startKey2, false, endKey1, false)); + Assert.assertEquals(keyUtf8.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + keyUtf8.getPartIds(new Row(startKey2), false, new Row(endKey1), false)); - Assert.assertEquals(keyUtf8.getPartIds(startKey1, true, endKey1, true), - keyUtf8.getPartIds(startKey2, true, endKey2, true)); + Assert.assertEquals(keyUtf8.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + keyUtf8.getPartIds(new Row(startKey2), true, new Row(endKey2), true)); - Assert.assertEquals(keyUtf8.getPartIds(startKey1, true, endKey2, true), - keyUtf8.getPartIds(startKey2, true, endKey1, true)); + Assert.assertEquals(keyUtf8.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + keyUtf8.getPartIds(new Row(startKey2), true, new Row(endKey1), true)); - Assert.assertEquals(keyUtf8.getPartIds(startKey1, false, endKey2, false), - keyUtf8.getPartIds(startKey2, false, endKey1, false)); + Assert.assertEquals(keyUtf8.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + keyUtf8.getPartIds(new Row(startKey2), false, new Row(endKey1), false)); - Assert.assertNotEquals(keyUtf8.getPartIds(startKey1, true, endKey1, true), - keyUtf8.getPartIds(startKey4, true, endKey4, true)); + Assert.assertNotEquals( + keyUtf8.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + keyUtf8.getPartIds(new Row(startKey4), true, new Row(endKey4), true)); - Assert.assertNotEquals(keyUtf8.getPartIds(startKey2, true, endKey2, true), - keyUtf8.getPartIds(startKey5, true, endKey5, true)); + Assert.assertNotEquals( + keyUtf8.getPartIds(new Row(startKey2), true, new Row(endKey2), true), + keyUtf8.getPartIds(new Row(startKey5), true, new Row(endKey5), true)); - Assert.assertNotEquals(keyUtf8.getPartIds(startKey3, false, endKey3, false), - keyUtf8.getPartIds(startKey6, false, endKey6, false)); + Assert.assertNotEquals( + keyUtf8.getPartIds(new Row(startKey3), false, new Row(endKey3), false), + keyUtf8.getPartIds(new Row(startKey6), false, new Row(endKey6), false)); } diff --git a/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDescTest.java b/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDescTest.java index 35637a24..9f2cf6f7 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDescTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/location/model/partition/ObRangePartDescTest.java @@ -22,6 +22,7 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.ObCollationType; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObColumn; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; +import com.alipay.oceanbase.rpc.mutation.Row; import com.alipay.oceanbase.rpc.protocol.payload.impl.column.ObSimpleColumn; import org.junit.Assert; import org.junit.Before; @@ -29,6 +30,8 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.List; import static com.alipay.oceanbase.rpc.location.model.partition.ObPartFuncType.RANGE_COLUMNS; @@ -140,78 +143,160 @@ public void setUp() { @Test public void testGetPartId() { - long partId = rangeBinary.getPartId("partition_1", "column_1", System.currentTimeMillis()); + Map partition_1 = new HashMap() { + { + put("K", "partition_1"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map partition_2 = new HashMap() { + { + put("K", "partition_2"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + long partId = rangeBinary.getPartId(new Row(partition_1)); Assert.assertEquals(1, partId); - partId = rangeBinary.getPartId("a", "column_1", System.currentTimeMillis()); + Map test_a = new HashMap() { + { + put("K", "a"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + partId = rangeBinary.getPartId(new Row(test_a)); Assert.assertEquals(0, partId); - partId = rangeBinary.getPartId("x", "column_1", System.currentTimeMillis()); + Map test_x = new HashMap() { + { + put("K", "x"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + partId = rangeBinary.getPartId(new Row(test_x)); Assert.assertEquals(2, partId); - Assert.assertEquals( - rangeBinary.getPartId("partition_1", "column_1", System.currentTimeMillis()), - rangeBinary.getPartId("partition_2", "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - rangeBinary.getPartId("test_1", "column_1", System.currentTimeMillis()), - rangeBinary.getPartId("test_2", "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - rangeBinary.getPartId("test_1", "column_1", System.currentTimeMillis()), - rangeBinary.getPartId("test_2".getBytes(), "column_1", System.currentTimeMillis())); - - Assert.assertEquals( - rangeBinary.getPartId("test_1".getBytes(), "column_1", System.currentTimeMillis()), - rangeBinary.getPartId("test_2".getBytes(), "column_1", System.currentTimeMillis())); - - partId = rangeBinary.getPartId("A", "column_1", System.currentTimeMillis()); + Assert.assertEquals(rangeBinary.getPartId(new Row(partition_1)), + rangeBinary.getPartId(new Row(partition_2))); + + Map test_1 = new HashMap() { + { + put("K", "test_1"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map test_1_bytes = new HashMap() { + { + put("K", "test_1".getBytes()); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map test_2 = new HashMap() { + { + put("K", "test_2"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map test_2_bytes = new HashMap() { + { + put("K", "test_2".getBytes()); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + + Assert.assertEquals(rangeBinary.getPartId(new Row(test_1)), + rangeBinary.getPartId(new Row(test_2))); + + Assert.assertEquals(rangeBinary.getPartId(new Row(test_1)), + rangeBinary.getPartId(new Row(test_2_bytes))); + + Assert.assertEquals(rangeBinary.getPartId(new Row(test_1_bytes)), + rangeBinary.getPartId(new Row(test_2_bytes))); + + Map test_A = new HashMap() { + { + put("K", "A"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map test_P = new HashMap() { + { + put("K", "P"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map test_X = new HashMap() { + { + put("K", "X"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + Map test_p = new HashMap() { + { + put("K", "p"); + put("Q", "column_1"); + put("T", System.currentTimeMillis()); + } + }; + + partId = rangeBinary.getPartId(new Row(test_A)); Assert.assertEquals(0, partId); - partId = rangeBinary.getPartId("P", "column_1", System.currentTimeMillis()); + partId = rangeBinary.getPartId(new Row(test_P)); Assert.assertEquals(0, partId); - partId = rangeBinary.getPartId("X", "column_1", System.currentTimeMillis()); + partId = rangeBinary.getPartId(new Row(test_X)); Assert.assertEquals(0, partId); - partId = rangeUtf8_CI.getPartId("a", "column_1", System.currentTimeMillis()); + partId = rangeUtf8_CI.getPartId(new Row(test_a)); Assert.assertEquals(0, partId); - partId = rangeUtf8_CI.getPartId("partition_1", "column_1", System.currentTimeMillis()); + partId = rangeUtf8_CI.getPartId(new Row(partition_1)); Assert.assertEquals(1, partId); - partId = rangeUtf8_CI.getPartId("x", "column_1", System.currentTimeMillis()); + partId = rangeUtf8_CI.getPartId(new Row(test_x)); Assert.assertEquals(2, partId); - partId = rangeUtf8_CI.getPartId("A", "column_1", System.currentTimeMillis()); + partId = rangeUtf8_CI.getPartId(new Row(test_A)); Assert.assertEquals(0, partId); - partId = rangeUtf8_CI.getPartId("P", "column_1", System.currentTimeMillis()); + partId = rangeUtf8_CI.getPartId(new Row(test_P)); Assert.assertEquals(1, partId); - partId = rangeUtf8_CI.getPartId("X", "column_1", System.currentTimeMillis()); + partId = rangeUtf8_CI.getPartId(new Row(test_X)); Assert.assertEquals(2, partId); - partId = rangeUtf8.getPartId("a", "column_1", System.currentTimeMillis()); + partId = rangeUtf8.getPartId(new Row(test_a)); Assert.assertEquals(0, partId); - partId = rangeUtf8.getPartId("p", "column_1", System.currentTimeMillis()); + partId = rangeUtf8.getPartId(new Row(test_p)); Assert.assertEquals(1, partId); - partId = rangeUtf8.getPartId("x", "column_1", System.currentTimeMillis()); + partId = rangeUtf8.getPartId(new Row(test_x)); Assert.assertEquals(2, partId); - partId = rangeUtf8.getPartId("A", "column_1", System.currentTimeMillis()); + partId = rangeUtf8.getPartId(new Row(test_A)); Assert.assertEquals(0, partId); - partId = rangeUtf8.getPartId("P", "column_1", System.currentTimeMillis()); + partId = rangeUtf8.getPartId(new Row(test_P)); Assert.assertEquals(0, partId); - partId = rangeUtf8.getPartId("X", "column_1", System.currentTimeMillis()); + partId = rangeUtf8.getPartId(new Row(test_X)); Assert.assertEquals(0, partId); - ArrayList rowKeys = new ArrayList(); - rowKeys.add(new Object[] { "P", "column_1", System.currentTimeMillis() }); + ArrayList rowKeys = new ArrayList(); + rowKeys.add(new Row(test_P)); partId = rangeUtf8.getPartId(rowKeys, true); Assert.assertEquals(0, partId); Assert.assertTrue(rangeUtf8.toString().contains("partExpr")); @@ -220,137 +305,245 @@ public void testGetPartId() { @Test public void testGetPartIds() { long timestamp = System.currentTimeMillis(); - Object[] startKey1 = new Object[] { "partition_1", "column_1", timestamp }; - Object[] endKey1 = new Object[] { "partition_2", "column_1", timestamp }; - - Object[] startKey2 = new Object[] { "partition_1".getBytes(), "column_1", timestamp }; - Object[] endKey2 = new Object[] { "partition_2".getBytes(), "column_1", timestamp }; - - Object[] startKey3 = new Object[] { "yes_1".getBytes(), "column_1", timestamp }; - Object[] endKey3 = new Object[] { "yes_2".getBytes(), "column_1", timestamp }; - - Object[] startKey4 = new Object[] { "PARTITION_1", "column_1", timestamp }; - Object[] endKey4 = new Object[] { "PARTITION_2", "column_1", timestamp }; - - Object[] startKey5 = new Object[] { "PARTITION_1".getBytes(), "column_1", timestamp }; - Object[] endKey5 = new Object[] { "PARTITION_2".getBytes(), "column_1", timestamp }; - - Object[] startKey6 = new Object[] { "YES_1".getBytes(), "column_1", timestamp }; - Object[] endKey6 = new Object[] { "YES_2".getBytes(), "column_1", timestamp }; + Map startKey1 = new HashMap() { + { + put("K", "partition_1"); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey1 = new HashMap() { + { + put("K", "partition_1"); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey2 = new HashMap() { + { + put("K", "partition_1".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey2 = new HashMap() { + { + put("K", "partition_1".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey3 = new HashMap() { + { + put("K", "yes_1".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey3 = new HashMap() { + { + put("K", "yes_2".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey4 = new HashMap() { + { + put("K", "PARTITION_1"); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey4 = new HashMap() { + { + put("K", "PARTITION_2"); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey5 = new HashMap() { + { + put("K", "PARTITION_1".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey5 = new HashMap() { + { + put("K", "PARTITION_2".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + + Map startKey6 = new HashMap() { + { + put("K", "YES_1".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; + Map endKey6 = new HashMap() { + { + put("K", "YES_2".getBytes()); + put("Q", "column_1"); + put("T", timestamp); + } + }; - Assert.assertEquals(rangeBinary.getPartIds(startKey1, true, endKey1, true), - rangeBinary.getPartIds(startKey2, true, endKey2, true)); + Assert.assertEquals( + rangeBinary.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + rangeBinary.getPartIds(new Row(startKey2), true, new Row(endKey2), true)); - Assert.assertEquals(rangeBinary.getPartIds(startKey1, true, endKey2, true), - rangeBinary.getPartIds(startKey2, true, endKey1, true)); + Assert.assertEquals( + rangeBinary.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + rangeBinary.getPartIds(new Row(startKey2), true, new Row(endKey1), true)); - Assert.assertEquals(rangeBinary.getPartIds(startKey1, false, endKey2, false), - rangeBinary.getPartIds(startKey2, false, endKey1, false)); + Assert.assertEquals( + rangeBinary.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + rangeBinary.getPartIds(new Row(startKey2), false, new Row(endKey1), false)); - Assert.assertNotEquals(rangeBinary.getPartIds(startKey1, true, endKey1, true), - rangeBinary.getPartIds(startKey4, true, endKey4, true)); + Assert.assertNotEquals( + rangeBinary.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + rangeBinary.getPartIds(new Row(startKey4), true, new Row(endKey4), true)); - Assert.assertNotEquals(rangeBinary.getPartIds(startKey2, true, endKey2, true), - rangeBinary.getPartIds(startKey5, true, endKey5, true)); + Assert.assertNotEquals( + rangeBinary.getPartIds(new Row(startKey2), true, new Row(endKey2), true), + rangeBinary.getPartIds(new Row(startKey5), true, new Row(endKey5), true)); - Assert.assertNotEquals(rangeBinary.getPartIds(startKey3, false, endKey3, false), - rangeBinary.getPartIds(startKey6, false, endKey6, false)); + Assert.assertNotEquals( + rangeBinary.getPartIds(new Row(startKey3), false, new Row(endKey3), false), + rangeBinary.getPartIds(new Row(startKey6), false, new Row(endKey6), false)); - Assert.assertNotEquals(rangeBinary.getPartIds(startKey1, true, endKey1, true), - rangeBinary.getPartIds(startKey5, true, endKey5, true)); + Assert.assertNotEquals( + rangeBinary.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + rangeBinary.getPartIds(new Row(startKey5), true, new Row(endKey5), true)); - Assert.assertNotEquals(rangeBinary.getPartIds(startKey1, true, endKey2, true), - rangeBinary.getPartIds(startKey5, true, endKey4, true)); + Assert.assertNotEquals( + rangeBinary.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + rangeBinary.getPartIds(new Row(startKey5), true, new Row(endKey4), true)); - Assert.assertNotEquals(rangeBinary.getPartIds(startKey1, false, endKey2, false), - rangeBinary.getPartIds(startKey5, false, endKey5, false)); + Assert.assertNotEquals( + rangeBinary.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + rangeBinary.getPartIds(new Row(startKey5), false, new Row(endKey5), false)); List partIds = new ArrayList(); partIds.add(1L); partIds.add(2L); - Assert.assertEquals(partIds, rangeBinary.getPartIds(startKey1, false, endKey3, false)); + Assert.assertEquals(partIds, + rangeBinary.getPartIds(new Row(startKey1), false, new Row(endKey3), false)); partIds = new ArrayList(); partIds.add(0L); - Assert.assertEquals(partIds, rangeBinary.getPartIds(startKey4, false, endKey4, false)); + Assert.assertEquals(partIds, + rangeBinary.getPartIds(new Row(startKey4), false, new Row(endKey4), false)); - Assert.assertEquals(partIds, rangeBinary.getPartIds(startKey6, false, endKey4, true)); + Assert.assertEquals(partIds, + rangeBinary.getPartIds(new Row(startKey6), false, new Row(endKey4), true)); - Assert.assertEquals(0, rangeBinary.getPartIds(startKey3, false, endKey1, true).size()); + Assert.assertEquals(0, + rangeBinary.getPartIds(new Row(startKey3), false, new Row(endKey1), true).size()); - Assert.assertEquals(rangeUtf8_CI.getPartIds(startKey1, true, endKey1, true), - rangeUtf8_CI.getPartIds(startKey2, true, endKey2, true)); + Assert.assertEquals( + rangeUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + rangeUtf8_CI.getPartIds(new Row(startKey2), true, new Row(endKey2), true)); - Assert.assertEquals(rangeUtf8_CI.getPartIds(startKey1, true, endKey2, true), - rangeUtf8_CI.getPartIds(startKey2, true, endKey1, true)); + Assert.assertEquals( + rangeUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + rangeUtf8_CI.getPartIds(new Row(startKey2), true, new Row(endKey1), true)); - Assert.assertEquals(rangeUtf8_CI.getPartIds(startKey1, false, endKey2, false), - rangeUtf8_CI.getPartIds(startKey2, false, endKey1, false)); + Assert.assertEquals( + rangeUtf8_CI.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + rangeUtf8_CI.getPartIds(new Row(startKey2), false, new Row(endKey1), false)); - Assert.assertEquals(rangeUtf8_CI.getPartIds(startKey1, true, endKey1, true), - rangeUtf8_CI.getPartIds(startKey4, true, endKey4, true)); + Assert.assertEquals( + rangeUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + rangeUtf8_CI.getPartIds(new Row(startKey4), true, new Row(endKey4), true)); - Assert.assertEquals(rangeUtf8_CI.getPartIds(startKey2, true, endKey2, true), - rangeUtf8_CI.getPartIds(startKey5, true, endKey5, true)); + Assert.assertEquals( + rangeUtf8_CI.getPartIds(new Row(startKey2), true, new Row(endKey2), true), + rangeUtf8_CI.getPartIds(new Row(startKey5), true, new Row(endKey5), true)); - Assert.assertEquals(rangeUtf8_CI.getPartIds(startKey3, false, endKey3, false), - rangeUtf8_CI.getPartIds(startKey6, false, endKey6, false)); + Assert.assertEquals( + rangeUtf8_CI.getPartIds(new Row(startKey3), false, new Row(endKey3), false), + rangeUtf8_CI.getPartIds(new Row(startKey6), false, new Row(endKey6), false)); - Assert.assertEquals(rangeUtf8_CI.getPartIds(startKey1, true, endKey1, true), - rangeUtf8_CI.getPartIds(startKey5, true, endKey5, true)); + Assert.assertEquals( + rangeUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + rangeUtf8_CI.getPartIds(new Row(startKey5), true, new Row(endKey5), true)); - Assert.assertEquals(rangeUtf8_CI.getPartIds(startKey1, true, endKey2, true), - rangeUtf8_CI.getPartIds(startKey5, true, endKey4, true)); + Assert.assertEquals( + rangeUtf8_CI.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + rangeUtf8_CI.getPartIds(new Row(startKey5), true, new Row(endKey4), true)); - Assert.assertEquals(rangeUtf8_CI.getPartIds(startKey1, false, endKey2, false), - rangeUtf8_CI.getPartIds(startKey5, false, endKey4, false)); + Assert.assertEquals( + rangeUtf8_CI.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + rangeUtf8_CI.getPartIds(new Row(startKey5), false, new Row(endKey4), false)); partIds = new ArrayList(); partIds.add(1L); partIds.add(2L); - Assert.assertEquals(partIds, rangeUtf8_CI.getPartIds(startKey1, false, endKey3, false)); - Assert.assertEquals(partIds, rangeUtf8_CI.getPartIds(startKey4, false, endKey6, false)); - Assert.assertEquals(0, rangeUtf8_CI.getPartIds(startKey3, false, endKey1, true).size()); + Assert.assertEquals(partIds, + rangeUtf8_CI.getPartIds(new Row(startKey1), false, new Row(endKey3), false)); + Assert.assertEquals(partIds, + rangeUtf8_CI.getPartIds(new Row(startKey4), false, new Row(endKey6), false)); + Assert.assertEquals(0, + rangeUtf8_CI.getPartIds(new Row(startKey3), false, new Row(endKey1), true).size()); - Assert.assertEquals(rangeUtf8.getPartIds(startKey1, true, endKey1, true), - rangeUtf8.getPartIds(startKey2, true, endKey2, true)); + Assert.assertEquals(rangeUtf8.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + rangeUtf8.getPartIds(new Row(startKey2), true, new Row(endKey2), true)); - Assert.assertEquals(rangeUtf8.getPartIds(startKey1, true, endKey2, true), - rangeUtf8.getPartIds(startKey2, true, endKey1, true)); + Assert.assertEquals(rangeUtf8.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + rangeUtf8.getPartIds(new Row(startKey2), true, new Row(endKey1), true)); - Assert.assertEquals(rangeUtf8.getPartIds(startKey1, false, endKey2, false), - rangeUtf8.getPartIds(startKey2, false, endKey1, false)); + Assert.assertEquals( + rangeUtf8.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + rangeUtf8.getPartIds(new Row(startKey2), false, new Row(endKey1), false)); - Assert.assertNotEquals(rangeUtf8.getPartIds(startKey1, true, endKey1, true), - rangeUtf8.getPartIds(startKey4, true, endKey4, true)); + Assert.assertNotEquals( + rangeUtf8.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + rangeUtf8.getPartIds(new Row(startKey4), true, new Row(endKey4), true)); - Assert.assertNotEquals(rangeUtf8.getPartIds(startKey2, true, endKey2, true), - rangeUtf8.getPartIds(startKey5, true, endKey5, true)); + Assert.assertNotEquals( + rangeUtf8.getPartIds(new Row(startKey2), true, new Row(endKey2), true), + rangeUtf8.getPartIds(new Row(startKey5), true, new Row(endKey5), true)); - Assert.assertNotEquals(rangeUtf8.getPartIds(startKey3, false, endKey3, false), - rangeUtf8.getPartIds(startKey6, false, endKey6, false)); + Assert.assertNotEquals( + rangeUtf8.getPartIds(new Row(startKey3), false, new Row(endKey3), false), + rangeUtf8.getPartIds(new Row(startKey6), false, new Row(endKey6), false)); - Assert.assertNotEquals(rangeUtf8.getPartIds(startKey1, true, endKey1, true), - rangeUtf8.getPartIds(startKey5, true, endKey5, true)); + Assert.assertNotEquals( + rangeUtf8.getPartIds(new Row(startKey1), true, new Row(endKey1), true), + rangeUtf8.getPartIds(new Row(startKey5), true, new Row(endKey5), true)); - Assert.assertNotEquals(rangeUtf8.getPartIds(startKey1, true, endKey2, true), - rangeUtf8.getPartIds(startKey5, true, endKey4, true)); + Assert.assertNotEquals( + rangeUtf8.getPartIds(new Row(startKey1), true, new Row(endKey2), true), + rangeUtf8.getPartIds(new Row(startKey5), true, new Row(endKey4), true)); - Assert.assertNotEquals(rangeUtf8.getPartIds(startKey1, false, endKey2, false), - rangeUtf8.getPartIds(startKey5, false, endKey5, false)); + Assert.assertNotEquals( + rangeUtf8.getPartIds(new Row(startKey1), false, new Row(endKey2), false), + rangeUtf8.getPartIds(new Row(startKey5), false, new Row(endKey5), false)); partIds = new ArrayList(); partIds.add(1L); partIds.add(2L); - Assert.assertEquals(partIds, rangeUtf8.getPartIds(startKey1, false, endKey3, false)); + Assert.assertEquals(partIds, + rangeUtf8.getPartIds(new Row(startKey1), false, new Row(endKey3), false)); partIds = new ArrayList(); partIds.add(0L); - Assert.assertEquals(partIds, rangeUtf8.getPartIds(startKey4, false, endKey4, false)); + Assert.assertEquals(partIds, + rangeUtf8.getPartIds(new Row(startKey4), false, new Row(endKey4), false)); - Assert.assertEquals(partIds, rangeUtf8.getPartIds(startKey6, false, endKey4, true)); + Assert.assertEquals(partIds, + rangeUtf8.getPartIds(new Row(startKey6), false, new Row(endKey4), true)); - Assert.assertEquals(0, rangeUtf8.getPartIds(startKey3, false, endKey1, true).size()); + Assert.assertEquals(0, + rangeUtf8.getPartIds(new Row(startKey3), false, new Row(endKey1), true).size()); } diff --git a/src/test/java/com/alipay/oceanbase/rpc/protocol/codec/ObRpcPacketHeaderTest.java b/src/test/java/com/alipay/oceanbase/rpc/protocol/codec/ObRpcPacketHeaderTest.java index 209fd592..0cb6f9b8 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/protocol/codec/ObRpcPacketHeaderTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/protocol/codec/ObRpcPacketHeaderTest.java @@ -331,7 +331,7 @@ private void checkObRpcPacketHeaderObRpcCostTime(ObRpcPacketHeader obRpcPacketHe } private void checkObRpcPacketHeaderClusterId(ObRpcPacketHeader obRpcPacketHeader) { - Assert.assertEquals(obRpcPacketHeader.getClusterId(), clusterId); + Assert.assertEquals(obRpcPacketHeader.getDstClusterId(), clusterId); } private void checkObRpcPacketHeaderCompress(ObRpcPacketHeader obRpcPacketHeader) { diff --git a/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationResultTest.java b/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationResultTest.java index 109f966d..efeca50b 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationResultTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableBatchOperationResultTest.java @@ -67,7 +67,7 @@ public void test() { assertEquals(123, deResult.getHeader().getErrno()); assertEquals(new String("HY0001".getBytes()), new String(deResult.getHeader().getSqlState())); - assertEquals(new String("msg".getBytes()), new String(deResult.getHeader().getMsg())); + assertEquals(new String("ms".getBytes()), deResult.getHeader().getErrMsg()); assertEquals(ObTableOperationType.GET, deResult.getOperationType()); ObITableEntity deEntity = deResult.getEntity(); assertNotNull(deEntity); diff --git a/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationResultTest.java b/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationResultTest.java index 0b04d598..ee77c08c 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationResultTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/ObTableOperationResultTest.java @@ -61,7 +61,7 @@ public void test() { assertEquals(123, deResult.getHeader().getErrno()); assertEquals(new String("HY0001".getBytes()), new String(deResult.getHeader().getSqlState())); - assertEquals(new String("msg".getBytes()), new String(deResult.getHeader().getMsg())); + assertEquals(new String("ms".getBytes()), deResult.getHeader().getErrMsg()); assertEquals(ObTableOperationType.GET, deResult.getOperationType()); assertEquals(10000, deResult.getAffectedRows()); ObITableEntity deEntity = deResult.getEntity(); diff --git a/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryAsyncPayloadTest.java b/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryAsyncPayloadTest.java new file mode 100644 index 00000000..d6f477fd --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/protocol/payload/impl/execute/query/ObTableQueryAsyncPayloadTest.java @@ -0,0 +1,151 @@ +/*- + * #%L + * OBKV Table Client Framework + * %% + * Copyright (C) 2021 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ +package com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query; + +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObCollationLevel; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObCollationType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObj; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjMeta; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObObjType; +import com.alipay.oceanbase.rpc.protocol.payload.impl.ObRowKey; +import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncResult; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.PooledByteBufAllocator; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ObTableQueryAsyncPayloadTest { + @Test + public void test_ObTableQueryAsync() { + ObTableQuery obTableQuery = getObTableQuery(); + byte[] bytes = obTableQuery.encode(); + ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(); + buf.writeBytes(bytes); + + ObTableQuery obTableQuery1 = getObTableQuery(); + obTableQuery1.decode(buf); + + checkObTableQuery(obTableQuery, obTableQuery1); + } + + @Test + public void test_ObTableQueryAsyncResult() { + ObTableQueryAsyncResult obTableQueryAsyncResult = new ObTableQueryAsyncResult(); + ObTableQueryResult obTableQueryResult = new ObTableQueryResult(); + obTableQueryResult.setRowCount(0); + obTableQueryResult.addPropertiesName("test1"); + obTableQueryResult.addPropertiesName("test2"); + obTableQueryAsyncResult.setAffectedEntity(obTableQueryResult); + obTableQueryAsyncResult.setEnd(true); + obTableQueryAsyncResult.setSessionId(0); + + byte[] bytes = obTableQueryAsyncResult.encode(); + ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(); + buf.writeBytes(bytes); + + ObTableQueryAsyncResult newObTableQueryAsyncResult = new ObTableQueryAsyncResult(); + ObTableQueryResult newObTableQueryResult = new ObTableQueryResult(); + newObTableQueryAsyncResult.setAffectedEntity(newObTableQueryResult); + newObTableQueryAsyncResult.decode(buf); + + assertEquals(obTableQueryResult.getRowCount(), newObTableQueryResult.getRowCount()); + assertEquals(obTableQueryResult.getPropertiesNames().size(), newObTableQueryResult + .getPropertiesNames().size()); + assertEquals(obTableQueryResult.getPropertiesNames().get(0), newObTableQueryResult + .getPropertiesNames().get(0)); + assertEquals(obTableQueryResult.getPropertiesNames().get(1), newObTableQueryResult + .getPropertiesNames().get(1)); + buf.release(); + + } + + private ObTableQuery getObTableQuery() { + ObTableQuery obTableQuery = new ObTableQuery(); + obTableQuery.addKeyRange(getObNewRange()); + obTableQuery.addKeyRange(getObNewRange()); + obTableQuery.addSelectColumn("test_selectColumn"); + obTableQuery.setFilterString("test_filterString"); + obTableQuery.setIndexName("test_indexName"); + obTableQuery.setLimit(100); + obTableQuery.setOffset(200); + obTableQuery.setScanOrder(ObScanOrder.Forward); + + return obTableQuery; + } + + private void checkObTableQuery(ObTableQuery obTableQueryAsync, ObTableQuery newObTableQueryAsync) { + ObTableQuery obTableQuery = obTableQueryAsync; + ObTableQuery obTableQuery1 = newObTableQueryAsync; + checkObNewRange(obTableQuery.getKeyRanges().get(0), obTableQuery1.getKeyRanges().get(0)); + assertEquals(obTableQuery.getSelectColumns().get(0), obTableQuery1.getSelectColumns() + .get(0)); + assertEquals(obTableQuery.getFilterString(), obTableQuery1.getFilterString()); + assertEquals(obTableQuery.getIndexName(), obTableQuery1.getIndexName()); + assertEquals(obTableQuery.getLimit(), obTableQuery1.getLimit()); + assertEquals(obTableQuery.getOffset(), obTableQuery1.getOffset()); + assertEquals(obTableQuery.getScanOrder(), obTableQuery1.getScanOrder()); + + checkObNewRange(obTableQuery.getKeyRanges().get(1), obTableQuery1.getKeyRanges().get(1)); + assertEquals(obTableQuery.getSelectColumns().get(0), obTableQuery1.getSelectColumns() + .get(0)); + assertEquals(obTableQuery.getFilterString(), obTableQuery1.getFilterString()); + assertEquals(obTableQuery.getIndexName(), obTableQuery1.getIndexName()); + assertEquals(obTableQuery.getLimit(), obTableQuery1.getLimit()); + assertEquals(obTableQuery.getOffset(), obTableQuery1.getOffset()); + assertEquals(obTableQuery.getScanOrder(), obTableQuery1.getScanOrder()); + } + + private ObNewRange getObNewRange() { + ObNewRange obNewRange = new ObNewRange(); + obNewRange.setTableId(100); + + ObRowKey rowKey = new ObRowKey(); + ObObj obj = new ObObj(); + obj.setMeta(new ObObjMeta(ObObjType.ObInt64Type, ObCollationLevel.CS_LEVEL_EXPLICIT, + ObCollationType.CS_TYPE_BINARY, (byte) 10)); + obj.setValue(12345); + rowKey.addObj(obj); + obj = new ObObj(); + obj.setMeta(new ObObjMeta(ObObjType.ObInt64Type, ObCollationLevel.CS_LEVEL_EXPLICIT, + ObCollationType.CS_TYPE_BINARY, (byte) 10)); + obj.setValue(45678); + rowKey.addObj(obj); + + obNewRange.setStartKey(rowKey); + obNewRange.setEndKey(rowKey); + return obNewRange; + } + + private void checkObNewRange(ObNewRange obNewRange, ObNewRange newObNewRange) { + assertEquals(obNewRange.getTableId(), newObNewRange.getTableId()); + assertEquals(obNewRange.getBorderFlag(), newObNewRange.getBorderFlag()); + + assertEquals(obNewRange.getStartKey().getObjCount(), newObNewRange.getStartKey() + .getObjCount()); + assertEquals(Integer.valueOf(obNewRange.getStartKey().getObj(0).getValue() + ""), + Integer.valueOf(newObNewRange.getStartKey().getObj(0).getValue() + "")); + assertEquals(Integer.valueOf(obNewRange.getStartKey().getObj(1).getValue() + ""), + Integer.valueOf(newObNewRange.getStartKey().getObj(1).getValue() + "")); + + assertEquals(obNewRange.getEndKey().getObjCount(), newObNewRange.getEndKey().getObjCount()); + assertEquals(Integer.valueOf(obNewRange.getEndKey().getObj(0).getValue() + ""), + Integer.valueOf(newObNewRange.getEndKey().getObj(0).getValue() + "")); + assertEquals(Integer.valueOf(obNewRange.getEndKey().getObj(1).getValue() + ""), + Integer.valueOf(newObNewRange.getEndKey().getObj(1).getValue() + "")); + } +} \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/rpc/table/ObTableConnectionTest.java b/src/test/java/com/alipay/oceanbase/rpc/table/ObTableConnectionTest.java index b6049547..c508c483 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/table/ObTableConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/table/ObTableConnectionTest.java @@ -48,22 +48,50 @@ public void setup() throws Exception { @Test public void testVarcharConcurrent() throws Exception { - test_varchar_helper_thread("T101", 100); - test_varchar_helper_thread("T102", 100); - test_varchar_helper_thread("T103", 100); + // todo: only support in 3.x currently + if (ObTableClientTestUtil.isOBVersionGreaterEqualThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.setMetadataRefreshInterval(100); + obTableClient.addProperty(Property.SERVER_CONNECTION_POOL_SIZE.getKey(), + Integer.toString(TEST_CONNECTION_POOL_SIZE)); + obTableClient.init(); + syncRefreshMetaHelper(obTableClient); + + test_varchar_helper_thread(obTableClient, "T101", 100); + test_varchar_helper_thread(obTableClient, "T102", 100); + test_varchar_helper_thread(obTableClient, "T103", 100); } @Test public void testConnectionPoolSize() throws Exception { - ObPair obPair = obTableClient.getTable("test_varchar_table", - new String[] { "abc" }, false, false); - int poolSize = obPair.getRight().getObTableConnectionPoolSize(); - assertEquals(TEST_CONNECTION_POOL_SIZE, poolSize); + // todo: only support in 3.x currently + if (ObTableClientTestUtil.isOBVersionGreaterEqualThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + if (obTableClient.isOdpMode()) { + assertEquals(TEST_CONNECTION_POOL_SIZE, obTableClient.getOdpTable() + .getObTableConnectionPoolSize()); + } else { + ObPair obPair = obTableClient.getTable("test_varchar_table", + new String[] { "abc" }, false, false); + int poolSize = obPair.getRight().getObTable().getObTableConnectionPoolSize(); + assertEquals(TEST_CONNECTION_POOL_SIZE, poolSize); + } } @Test public void testWatermarkSetting() throws Exception { + // todo: only support in 3.x currently + if (ObTableClientTestUtil.isOBVersionGreaterEqualThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.addProperty(Property.NETTY_BUFFER_LOW_WATERMARK.getKey(), Integer.toString(TEST_NETTY_LOW_WATERMARK)); obTableClient.addProperty(Property.NETTY_BUFFER_HIGH_WATERMARK.getKey(), @@ -72,27 +100,45 @@ public void testWatermarkSetting() throws Exception { Integer.toString(TEST_NETTY_WAIT_INTERVAL)); obTableClient.init(); - ObPair obPair = obTableClient.getTable("test_varchar_table", - new String[] { "abc" }, false, false); - int lowWatermark = obPair.getRight().getNettyBufferLowWatermark(); - int highWatermark = obPair.getRight().getNettyBufferHighWatermark(); - int waitInterval = obPair.getRight().getNettyBlockingWaitInterval(); + if (obTableClient.isOdpMode()) { + assertEquals(TEST_NETTY_LOW_WATERMARK, obTableClient.getOdpTable() + .getNettyBufferLowWatermark()); + assertEquals(TEST_NETTY_HIGH_WATERMARK, obTableClient.getOdpTable() + .getNettyBufferHighWatermark()); + assertEquals(TEST_NETTY_WAIT_INTERVAL, obTableClient.getOdpTable() + .getNettyBlockingWaitInterval()); + } else { + ObPair obPair = obTableClient.getTable("test_varchar_table", + new String[] { "abc" }, false, false); + int lowWatermark = obPair.getRight().getObTable().getNettyBufferLowWatermark(); + int highWatermark = obPair.getRight().getObTable().getNettyBufferHighWatermark(); + int waitInterval = obPair.getRight().getObTable().getNettyBlockingWaitInterval(); - assertEquals(TEST_NETTY_LOW_WATERMARK, lowWatermark); - assertEquals(TEST_NETTY_HIGH_WATERMARK, highWatermark); - assertEquals(TEST_NETTY_WAIT_INTERVAL, waitInterval); + assertEquals(TEST_NETTY_LOW_WATERMARK, lowWatermark); + assertEquals(TEST_NETTY_HIGH_WATERMARK, highWatermark); + assertEquals(TEST_NETTY_WAIT_INTERVAL, waitInterval); + } } @Test public void testDefaultWatermark() throws Exception { - ObPair obPair = obTableClient.getTable("test_varchar_table", - new String[] { "abc" }, false, false); - int lowWatermark = obPair.getRight().getNettyBufferLowWatermark(); - int highWatermark = obPair.getRight().getNettyBufferHighWatermark(); - int waitInterval = obPair.getRight().getNettyBlockingWaitInterval(); - - assertEquals(Property.NETTY_BUFFER_LOW_WATERMARK.getDefaultInt(), lowWatermark); - assertEquals(Property.NETTY_BUFFER_HIGH_WATERMARK.getDefaultInt(), highWatermark); - assertEquals(Property.NETTY_BLOCKING_WAIT_INTERVAL.getDefaultInt(), waitInterval); + // todo: only support in 3.x currently + if (ObTableClientTestUtil.isOBVersionGreaterEqualThan(ObTableClientTestUtil.obVsn4000)) { + return; + } + + if (obTableClient.isOdpMode()) { + // do nothing + } else { + ObPair obPair = obTableClient.getTable("test_varchar_table", + new String[] { "abc" }, false, false); + int lowWatermark = obPair.getRight().getObTable().getNettyBufferLowWatermark(); + int highWatermark = obPair.getRight().getObTable().getNettyBufferHighWatermark(); + int waitInterval = obPair.getRight().getObTable().getNettyBlockingWaitInterval(); + + assertEquals(Property.NETTY_BUFFER_LOW_WATERMARK.getDefaultInt(), lowWatermark); + assertEquals(Property.NETTY_BUFFER_HIGH_WATERMARK.getDefaultInt(), highWatermark); + assertEquals(Property.NETTY_BLOCKING_WAIT_INTERVAL.getDefaultInt(), waitInterval); + } } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/temp/Crc64Util.java b/src/test/java/com/alipay/oceanbase/rpc/temp/Crc64Util.java index ad022e07..09d96e90 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/temp/Crc64Util.java +++ b/src/test/java/com/alipay/oceanbase/rpc/temp/Crc64Util.java @@ -26,7 +26,6 @@ public class Crc64Util { /** * CRC-64-ECMA-182 * - * @url http://en.wikipedia.org/wiki/Cyclic_redundancy_check * @url http://reveng.sourceforge.net/crc-catalogue/17plus.htm */ static final long[] CRC64TABLE = new long[] { 0L, 121597189939003392L, 243194379878006784L, diff --git a/src/test/java/com/alipay/oceanbase/rpc/util/ObHashUtilTest.java b/src/test/java/com/alipay/oceanbase/rpc/util/ObHashUtilTest.java index 2e72f9af..add97ada 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/util/ObHashUtilTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/util/ObHashUtilTest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.rpc.util; +import com.alipay.oceanbase.rpc.location.model.partition.ObPartFuncType; import com.alipay.oceanbase.rpc.protocol.payload.impl.ObCollationType; import com.alipay.oceanbase.rpc.util.hash.MurmurHash; import com.alipay.oceanbase.rpc.util.hash.ObHashSortUtf8mb4; @@ -32,17 +33,26 @@ public class ObHashUtilTest { public void testHash() { try { String s1 = new String("蚂蚁金服".getBytes("UTF-8"), "UTF-8"); - Assert.assertEquals(4331328712007588056L, - ObHashUtils.varcharHash(s1, ObCollationType.CS_TYPE_UTF8MB4_GENERAL_CI, 0)); + Assert.assertEquals(5857140318165389032L, ObHashUtils.varcharHash(s1, + ObCollationType.CS_TYPE_UTF8MB4_GENERAL_CI, 0, ObPartFuncType.KEY_V3)); } catch (UnsupportedEncodingException e) { Assert.fail("Don't support utf8 encoding."); } + Assert.assertEquals(-6632533813747801170L, ObHashUtils.varcharHash("partitionKey", + ObCollationType.CS_TYPE_BINARY, 0, ObPartFuncType.KEY_V3)); - Assert.assertEquals(8342516951305938535L, - ObHashUtils.varcharHash("abcdef", ObCollationType.CS_TYPE_UTF8MB4_GENERAL_CI, 0)); + Assert.assertEquals(3972054043075874492L, ObHashUtils.longHash(145, 0)); - Assert.assertEquals(3797055900148298469L, ObHashUtils.varcharHash(new byte[] { 0x01, 0x01, - 0x02, 0x7a, 0x71, 0x2b, 0x5f, 0x30 }, ObCollationType.CS_TYPE_UTF8MB4_BIN, 47)); + Assert.assertEquals(5483367784116059036L, ObHashUtils.varcharHash("abcdef", + ObCollationType.CS_TYPE_UTF8MB4_GENERAL_CI, 0, ObPartFuncType.KEY_V3)); + + Assert.assertEquals(141008085297877540L, ObHashUtils.varcharHash( + "Test Partition Key 0 c1 column c1 c1 c1", ObCollationType.CS_TYPE_UTF8MB4_BIN, 0, + ObPartFuncType.KEY_V3)); + + Assert.assertEquals(-8699342424260018756L, ObHashUtils.varcharHash(new byte[] { 0x01, 0x01, + 0x02, 0x7a, 0x71, 0x2b, 0x5f, 0x30 }, ObCollationType.CS_TYPE_UTF8MB4_BIN, 47, + ObPartFuncType.KEY_V3)); Assert.assertEquals(2452379837092619935L, ObHashUtils.longHash(754623846937L, 53)); @@ -84,13 +94,13 @@ public void testObHashSortUtf8mb4() { long seed = 0xc6a4a7935bd1e995L; long hashCode = 57; long res = ObHashSortUtf8mb4.obHashSortUtf8Mb4(new byte[] { (byte) 0xd3, (byte) 0x84 }, 2, - seed, hashCode); + seed, hashCode, false); Assert.assertEquals(7005345108782627091L, res); res = ObHashSortUtf8mb4.obHashSortUtf8Mb4(new byte[] { (byte) 0xe6, (byte) 0x84, - (byte) 0x57 }, 3, seed, hashCode); + (byte) 0x57 }, 3, seed, hashCode, false); Assert.assertEquals(-4132994306676758123L, res); res = ObHashSortUtf8mb4.obHashSortUtf8Mb4(new byte[] { (byte) 0xf6, (byte) 0x84, - (byte) 0x57, (byte) 0x6a, (byte) 0x43 }, 4, seed, hashCode); + (byte) 0x57, (byte) 0x6a, (byte) 0x43 }, 4, seed, hashCode, false); Assert.assertEquals(-4132994306676758123L, res); } diff --git a/src/test/java/com/alipay/oceanbase/rpc/util/ObTableClientTestUtil.java b/src/test/java/com/alipay/oceanbase/rpc/util/ObTableClientTestUtil.java index f26db5fa..1144cafc 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/util/ObTableClientTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/rpc/util/ObTableClientTestUtil.java @@ -19,20 +19,112 @@ import com.alipay.oceanbase.rpc.ObTableClient; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.UUID; + +import static com.alipay.oceanbase.rpc.ObGlobal.OB_VERSION; +import static com.alipay.oceanbase.rpc.ObGlobal.calcVersion; + public class ObTableClientTestUtil { - public static String FULL_USER_NAME = "full-user-name"; - public static String PARAM_URL = "config-url"; - public static String PASSWORD = "password"; - public static String PROXY_SYS_USER_NAME = "sys-user-name"; - public static String PROXY_SYS_USER_ENC_PASSWORD = "sys-user-password"; + public static String FULL_USER_NAME = "full-user-name"; + public static String PARAM_URL = "config-url"; + public static String PASSWORD = "password"; + public static String PROXY_SYS_USER_NAME = "sys-user-name"; + public static String PROXY_SYS_USER_PASSWORD = "sys-user-password"; + + public static boolean USE_ODP = false; + public static String ODP_IP = "ip-addr"; + public static int ODP_PORT = 0; + public static String ODP_DATABASE = "database-name"; + + public static String JDBC_IP = ""; + public static String JDBC_PORT = ""; + public static String JDBC_DATABASE = "OCEANBASE"; + public static String JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + JDBC_DATABASE + "?" + + "rewriteBatchedStatements=TRUE&" + + "allowMultiQueries=TRUE&" + + "useLocalSessionState=TRUE&" + + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + + "connectTimeout=60000"; public static ObTableClient newTestClient() throws Exception { ObTableClient obTableClient = new ObTableClient(); - obTableClient.setFullUserName(FULL_USER_NAME); - obTableClient.setParamURL(PARAM_URL); - obTableClient.setPassword(PASSWORD); - obTableClient.setSysUserName(PROXY_SYS_USER_NAME); - obTableClient.setEncSysPassword(PROXY_SYS_USER_ENC_PASSWORD); + if (!USE_ODP) { + obTableClient.setFullUserName(FULL_USER_NAME); + obTableClient.setParamURL(PARAM_URL); + obTableClient.setPassword(PASSWORD); + obTableClient.setSysUserName(PROXY_SYS_USER_NAME); + obTableClient.setSysPassword(PROXY_SYS_USER_PASSWORD); + } else { + obTableClient.setOdpMode(true); + obTableClient.setFullUserName(FULL_USER_NAME); + obTableClient.setOdpAddr(ODP_IP); + obTableClient.setOdpPort(ODP_PORT); + obTableClient.setDatabase(ODP_DATABASE); + obTableClient.setPassword(PASSWORD); + } + return obTableClient; } + + public static String getTenantName() { + String[] parts = FULL_USER_NAME.split("@"); + if (parts.length > 1) { + String[] keywordParts = parts[1].split("#"); + if (keywordParts.length > 0) { + return keywordParts[0]; + } + } + return ""; + } + + public static Connection getConnection() throws SQLException { + String[] userNames = FULL_USER_NAME.split("#"); + return DriverManager.getConnection(JDBC_URL, userNames[0], PASSWORD); + } + + public static Connection getSysConnection() throws SQLException { + return DriverManager.getConnection(JDBC_URL, "root@sys", PASSWORD); + } + + public static void cleanTable(String tableName) throws Exception { + Connection connection = ObTableClientTestUtil.getConnection(); + Statement statement = connection.createStatement(); + statement.execute("delete from " + tableName); + } + + public static String generateRandomStringByUUID(int times) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < times; i++) { + sb.append(UUID.randomUUID().toString().replaceAll("-", "")); + } + return sb.toString(); + } + + public static boolean isOBVersionGreaterEqualThan(long targetVersion) { + return OB_VERSION >= targetVersion; + } + + public static boolean isOBVersionGreaterThan(long targetVersion) { + return OB_VERSION >= targetVersion; + } + + public static boolean isOBVersionLessEqualThan(long targetVersion) { + return OB_VERSION <= targetVersion; + } + + public static boolean isOBVersionLessThan(long targetVersion) { + return OB_VERSION <= targetVersion; + } + + public static long obVsn4000 = calcVersion(4, (short) 0, (byte) 0, (byte) 0); + + static { + System.setProperty("logging.path", System.getProperty("user.dir") + "/logs"); + } } diff --git a/src/test/java/com/alipay/oceanbase/rpc/util/ObTableHotkeyThrottleUtil.java b/src/test/java/com/alipay/oceanbase/rpc/util/ObTableHotkeyThrottleUtil.java new file mode 100644 index 00000000..97f342db --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/rpc/util/ObTableHotkeyThrottleUtil.java @@ -0,0 +1,420 @@ +/*- + * #%L + * com.oceanbase:obkv-table-client + * %% + * Copyright (C) 2021 - 2022 OceanBase + * %% + * OBKV Table Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.rpc.util; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.mutation.*; +import com.alipay.oceanbase.rpc.mutation.result.BatchOperationResult; +import com.alipay.oceanbase.rpc.mutation.result.MutationResult; +import com.alipay.oceanbase.rpc.property.Property; +import com.alipay.oceanbase.rpc.stream.QueryResultSet; +import com.alipay.oceanbase.rpc.table.api.Table; +import com.alipay.oceanbase.rpc.table.api.TableQuery; +import org.junit.Assert; + +import java.util.ArrayList; +import java.util.List; + +import static com.alipay.oceanbase.rpc.mutation.MutationFactory.*; + +public class ObTableHotkeyThrottleUtil extends Thread { + int threadIdx; + int testNum; + String tableName = null; + String[] rowKeyColumnName = null; + + public int threadNum; + public static List unitOperationTimes = null; + public static List unitBlockTimes = null; + public static List totalOperationTimes = null; + public static List totalBlockTimes = null; + + public enum TestType { + random, specifiedKey + } + + public enum OperationType { + insert, update, insertOrUpdate, query, queryAndMutate, batchOperation + } + + TestType testType; + OperationType operationType; + public Table client = null; + Row rowKey; + int throttleNum = 0; + int passNum = 0; + int batchSize = 64; + long startTime = 0; + int unitBlockTime = 0; + int unitOperationTime = 0; + + public void init(int threadNum, int threadIdx, long startTime, String tableName, + String[] rowKeyColumnName, TestType testType, OperationType operationType, + int testNum, Table client, int batchSize, ColumnValue... rowKeyColumnValues) + throws Exception { + System.setProperty("ob_table_min_rslist_refresh_interval_millis", "1"); + this.threadNum = threadNum; + this.threadIdx = threadIdx; + this.startTime = startTime; + this.tableName = tableName; + this.testNum = testNum; + switch (testType) { + case random: { + rowKey = null; + this.testType = testType; + break; + } + case specifiedKey: { + if (rowKeyColumnValues != null) { + rowKey = row(rowKeyColumnValues); + this.testType = testType; + } else { + throw new IllegalArgumentException("invalid row key pass into init"); + } + break; + } + default: + throw new IllegalArgumentException("invalid test type pass into init"); + } + + this.operationType = operationType; + this.batchSize = batchSize; + + if (null == client) { + final ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); + obTableClient.setMetadataRefreshInterval(100); + obTableClient.addProperty(Property.RPC_CONNECT_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.RPC_LOGIN_TIMEOUT.getKey(), "800"); + obTableClient.addProperty(Property.SERVER_CONNECTION_POOL_SIZE.getKey(), "1"); + obTableClient.addProperty(Property.RPC_EXECUTE_TIMEOUT.getKey(), "2000"); + obTableClient.init(); + + this.client = obTableClient; + syncRefreshMetaHelper(obTableClient); + } else { + this.client = client; + } + ((ObTableClient) this.client).addRowKeyElement(this.tableName, rowKeyColumnName); + + if (null == unitOperationTimes || threadIdx == 0) { + unitOperationTimes = new ArrayList(threadNum); + for (int i = 0; i < threadNum; ++i) { + unitOperationTimes.add(0); + } + } + if (null == unitBlockTimes || threadIdx == 0) { + unitBlockTimes = new ArrayList(threadNum); + for (int i = 0; i < threadNum; ++i) { + unitBlockTimes.add(0); + } + } + if (null == totalOperationTimes || threadIdx == 0) { + totalOperationTimes = new ArrayList(threadNum); + for (int i = 0; i < threadNum; ++i) { + totalOperationTimes.add(0); + } + } + if (null == totalBlockTimes || threadIdx == 0) { + totalBlockTimes = new ArrayList(threadNum); + for (int i = 0; i < threadNum; ++i) { + totalBlockTimes.add(0); + } + } + } + + @Override + public void run() { + try { + switch (testType) { + case random: + runRandom(); + break; + case specifiedKey: + runSpecifiedKey(); + break; + default: + System.out.println(Thread.currentThread().getName() + + " has no test type to run"); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void runRandom() throws Exception { + System.out.println(Thread.currentThread().getName() + " begin to run random test"); + for (int i = 0; i < testNum; ++i) { + long randomNum = (long) (Math.random() * 2000000); + String rowKeyString = "Test" + randomNum; + Row rowKey = row(colVal("c1", randomNum), colVal("c2", rowKeyString)); + switch (operationType) { + case insert: + insertTest(rowKey, colVal("c3", new byte[] { 1 }), colVal("c4", 0L)); + break; + case update: + updateTest(rowKey, colVal("c3", new byte[] { 1 }), colVal("c4", 0L)); + break; + case insertOrUpdate: + insertOrUpdateTest(rowKey, colVal("c3", new byte[] { 1 }), colVal("c4", 0L)); + break; + case query: + queryTest(rowKey); + break; + case queryAndMutate: + queryAndMutateTest(rowKey, colVal("c3", new byte[] { 1 }), colVal("c4", 0L)); + break; + case batchOperation: + batchOperationTest(); + break; + } + // record operation time for each 2s + if (System.currentTimeMillis() - startTime > 2000) { + unitOperationTimes.set(threadIdx, unitOperationTime); + unitBlockTimes.set(threadIdx, unitBlockTime); + unitOperationTime = 0; + unitBlockTime = 0; + while (System.currentTimeMillis() - startTime > 2000) + startTime += 2000; + } + } + totalOperationTimes.set(threadIdx, throttleNum + passNum); + totalBlockTimes.set(threadIdx, throttleNum); + } + + private void runSpecifiedKey() throws Exception { + System.out.println(Thread.currentThread().getName() + " begin to run specified key test"); + for (int i = 0; i < testNum; ++i) { + switch (operationType) { + case insert: + insertTest(this.rowKey, colVal("c3", new byte[] { 1 }), colVal("c4", 0L)); + break; + case update: + updateTest(this.rowKey, colVal("c3", new byte[] { 1 }), colVal("c4", 0L)); + break; + case insertOrUpdate: + insertOrUpdateTest(this.rowKey, colVal("c3", new byte[] { 1 }), + colVal("c4", 0L)); + break; + case query: + queryTest(this.rowKey); + break; + case queryAndMutate: + queryAndMutateTest(this.rowKey, colVal("c3", new byte[] { 1 }), + colVal("c4", 0L)); + break; + } + // record operation time for each 2s + if (System.currentTimeMillis() - startTime > 2000) { + unitOperationTimes.set(threadIdx, unitOperationTime); + unitBlockTimes.set(threadIdx, unitBlockTime); + unitOperationTime = 0; + unitBlockTime = 0; + while (System.currentTimeMillis() - startTime > 2000) + startTime += 2000; + } + } + totalOperationTimes.set(threadIdx, throttleNum + passNum); + totalBlockTimes.set(threadIdx, throttleNum); + } + + private void insertTest(Row rowkey, ColumnValue... columnValues) throws Exception { + try { + ++unitOperationTime; + MutationResult insertResult = client.insert(this.tableName).setRowKey(rowkey) + .addMutateColVal(columnValues).execute(); + ++passNum; + } catch (Exception e) { + if (e instanceof ObTableUnexpectedException) { + if (((ObTableUnexpectedException) e).getErrorCode() == -4039) { + ++throttleNum; + ++unitBlockTime; + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } + + } + + private void updateTest(Row rowkey, ColumnValue... columnValues) throws Exception { + try { + ++unitOperationTime; + MutationResult updateResult = client.update(this.tableName).setRowKey(rowKey) + .addMutateColVal(columnValues).execute(); + ++passNum; + } catch (Exception e) { + if (e instanceof ObTableUnexpectedException) { + if (((ObTableUnexpectedException) e).getErrorCode() == -4039) { + ++throttleNum; + ++unitBlockTime; + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } + } + + private void insertOrUpdateTest(Row rowkey, ColumnValue... columnValues) throws Exception { + try { + ++unitOperationTime; + MutationResult insertOrUpdateResult = client.insertOrUpdate(this.tableName) + .setRowKey(rowkey).addMutateColVal(columnValues).execute(); + ++passNum; + } catch (Exception e) { + if (e instanceof ObTableUnexpectedException) { + if (((ObTableUnexpectedException) e).getErrorCode() == -4039) { + ++throttleNum; + ++unitBlockTime; + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } + } + + private void queryTest(Row rowkey) throws Exception { + try { + ++unitOperationTime; + TableQuery tableQuery = client.query(this.tableName); + tableQuery.addScanRange(rowkey.getValues(), rowkey.getValues()); + tableQuery.select("c1", "c2", "c3", "c4"); + QueryResultSet result_ = tableQuery.execute(); + ++passNum; + } catch (Exception e) { + if (e instanceof ObTableUnexpectedException) { + if (((ObTableUnexpectedException) e).getErrorCode() == -4039) { + ++throttleNum; + ++unitBlockTime; + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } + } + + private void queryAndMutateTest(Row rowkey, ColumnValue... columnValues) throws Exception { + try { + ++unitOperationTime; + MutationResult updateResult = client.update(this.tableName).setRowKey(rowkey) + .setFilter(null).addMutateColVal(columnValues).execute(); + ++passNum; + } catch (Exception e) { + if (e instanceof ObTableUnexpectedException) { + if (((ObTableUnexpectedException) e).getErrorCode() == -4039) { + ++throttleNum; + ++unitBlockTime; + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } + } + + private List generateBatchOpertaionIoU() { + List rowList = new ArrayList<>(); + for (int i = 0; i < batchSize; ++i) { + long randomNum = (long) (Math.random() * 2000000); + String rowKeyString = "Test" + randomNum; + rowList.add(insertOrUpdate().setRowKey(colVal("c1", randomNum), colVal("c2", rowKeyString)) + .addMutateColVal(colVal("c3", new byte[] { 1 })) + .addMutateColVal(colVal("c4", randomNum))); + } + return rowList; + } + + private void batchOperationTest() throws Exception { + try { + ++unitOperationTime; + BatchOperationResult batchResult = client.batchOperation(this.tableName) + .addOperation(generateBatchOpertaionIoU()).execute(); + ++passNum; + } catch (Exception e) { + if (e instanceof ObTableUnexpectedException) { + if (((ObTableUnexpectedException) e).getErrorCode() == -4039) { + if (++throttleNum % 50 == 0) { + ++throttleNum; + ++unitBlockTime; + } + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } else { + e.printStackTrace(); + Assert.assertNull(e); + } + } + } + + public List getUnitOperationTimes() { + return unitOperationTimes; + } + + public List getUnitBlockTimes() { + return unitBlockTimes; + } + + public List getTotalOperationTimes() { + return totalOperationTimes; + } + + public List getTotalBlockTimes() { + return totalBlockTimes; + } + + public void syncRefreshMetaHelper(final ObTableClient obTableClient) { + if (obTableClient.isOdpMode()) { + // do noting + } else { + new Thread(new Runnable() { + @Override + public void run() { + for (int i = 0; i < 10; i++) { + try { + obTableClient.syncRefreshMetadata(); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(); + } + } + } + }).start(); + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/rpc/util/YcsbBenchTest.java b/src/test/java/com/alipay/oceanbase/rpc/util/YcsbBenchTest.java deleted file mode 100644 index 6a48c652..00000000 --- a/src/test/java/com/alipay/oceanbase/rpc/util/YcsbBenchTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/*- - * #%L - * OBKV Table Client Framework - * %% - * Copyright (C) 2021 OceanBase - * %% - * OBKV Table Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.rpc.util; - -import com.alipay.oceanbase.rpc.ObTableClient; -import com.alipay.oceanbase.rpc.table.ObTable; -import com.yahoo.ycsb.ByteIterator; -import com.yahoo.ycsb.DB; -import com.yahoo.ycsb.Status; -import com.yahoo.ycsb.StringByteIterator; -import org.junit.Assert; -import org.junit.Test; - -import java.util.*; - -public class YcsbBenchTest { - - private void testYcsb(DB db) { - HashMap insertValues = new HashMap(); - insertValues.put("c2", new StringByteIterator("value1")); - Status status = db.insert("test_varchar_table", "bench_foo1", insertValues); - Assert.assertEquals(Status.OK, status); - status = db.insert("test_varchar_table", "bench_foo2", insertValues); - Assert.assertEquals(Status.OK, status); - status = db.insert("test_varchar_table", "bench_foo3", insertValues); - Assert.assertEquals(Status.OK, status); - - HashMap readValues = new HashMap(); - Set fields = new HashSet(); - fields.add("c2"); - status = db.read("test_varchar_table", "bench_foo1", fields, readValues); - Assert.assertEquals(Status.OK, status); - Assert.assertEquals(1, readValues.size()); - Assert.assertEquals("value1", readValues.get("c2").toString()); - - HashMap updateValues = new HashMap(); - updateValues.put("c2", new StringByteIterator("value21")); - status = db.update("test_varchar_table", "bench_foo1", updateValues); - Assert.assertEquals(Status.OK, status); - status = db.read("test_varchar_table", "bench_foo1", fields, readValues); - Assert.assertEquals(Status.OK, status); - Assert.assertEquals(1, readValues.size()); - Assert.assertEquals("value21", readValues.get("c2").toString()); - - status = db.delete("test_varchar_table", "bench_foo1"); - Assert.assertEquals(Status.OK, status); - status = db.delete("test_varchar_table", "bar"); - Assert.assertEquals(Status.ERROR, status); - } - - @Test - public void testYscbBenchClient() { - Properties p = new Properties(); - p.setProperty(YcsbBenchClient.OB_TABLE_CLIENT_PARAM_URL, ObTableClientTestUtil.PARAM_URL); - p.setProperty(YcsbBenchClient.OB_TABLE_CLIENT_FULL_USER_NAME, - ObTableClientTestUtil.FULL_USER_NAME); - p.setProperty(YcsbBenchClient.OB_TABLE_CLIENT_PASSWORD, ObTableClientTestUtil.PASSWORD); - p.setProperty(YcsbBenchClient.OB_TABLE_CLIENT_SYS_USER_NAME, - ObTableClientTestUtil.PROXY_SYS_USER_NAME); - p.setProperty(YcsbBenchClient.OB_TABLE_CLIENT_SYS_ENC_PASSWORD, - ObTableClientTestUtil.PROXY_SYS_USER_ENC_PASSWORD); - YcsbBenchClient client = new YcsbBenchClient(); - client.setProperties(p); - try { - client.init(); - testYcsb(client); - Vector> scanValues = new Vector(); - Set fields = new HashSet(); - fields.add("c2"); - Status status = client.scan("test_varchar_table", "bench_foo", 2, fields, scanValues); - Assert.assertEquals(Status.OK, status); - Assert.assertEquals(2, scanValues.size()); - - } catch (Exception e) { - Assert.fail("YcsbBenchClient Hit Exception:" + e.getMessage()); - } - } - - @Test - public void testYscbBench() throws Exception { - ObTableClient obTableClient = ObTableClientTestUtil.newTestClient(); - obTableClient.init(); - ObTable obTable = obTableClient.getTable("test_varchar_table", new Object[] { "abc" }, - true, true).getRight(); - - Properties p = new Properties(); - p.setProperty(YcsbBench.DB_PROPERTY, obTable.getDatabase()); - p.setProperty(YcsbBench.HOST_PROPERTY, obTable.getIp()); - p.setProperty(YcsbBench.PASSWORD_PROPERTY, obTable.getPassword()); - p.setProperty(YcsbBench.PORT_PROPERTY, "" + obTable.getPort()); - p.setProperty(YcsbBench.TENANT_PROPERTY, obTable.getTenantName()); - p.setProperty(YcsbBench.USERNAME_PROPERTY, obTable.getUserName()); - - YcsbBench ycsbBench = new YcsbBench(); - ycsbBench.setProperties(p); - try { - ycsbBench.init(); - testYcsb(ycsbBench); - } catch (Exception e) { - Assert.fail("YcsbBench Hit Exception:" + e.getMessage()); - } - } -} diff --git a/src/test/resources/ci.sql b/src/test/resources/ci.sql new file mode 100644 index 00000000..0ebb37d5 --- /dev/null +++ b/src/test/resources/ci.sql @@ -0,0 +1,600 @@ +USE TEST; + +CREATE TABLE IF NOT EXISTS `test_varchar_table` ( + `c1` varchar(20) NOT NULL, + `c2` varchar(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_blob_table` ( + `c1` varchar(20) NOT NULL, + `c2` blob DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_longblob_table` ( + `c1` varchar(20) NOT NULL, + `c2` longblob DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_varchar_table_for_exception` ( + `c1` varchar(20) NOT NULL, + `c2` varchar(20) NOT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_tinyint_table` ( + `c1` varchar(20) NOT NULL, + `c2` tinyint(4) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_smallint_table` ( + `c1` varchar(20) NOT NULL, + `c2` smallint(8) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_int_table` ( + `c1` int(12) NOT NULL, + `c2` int(12) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_bigint_table` ( + `c1` bigint(20) NOT NULL, + `c2` bigint(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_double_table` ( + `c1` varchar(20) NOT NULL, + `c2` double DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_float_table` ( + `c1` varchar(20) NOT NULL, + `c2` float DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_varbinary_table` ( + `c1` varchar(20) NOT NULL, + `c2` varbinary(20) DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_timestamp_table` ( + `c1` varchar(20) NOT NULL, + `c2` timestamp(6) DEFAULT NULL, + `c3` timestamp(3) DEFAULT NULL, + `c4` timestamp DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `test_datetime_table` ( + `c1` varchar(20) NOT NULL, + `c2` datetime(6) DEFAULT NULL, + `c3` datetime(3) DEFAULT NULL, + `c4` datetime DEFAULT NULL, + PRIMARY KEY (`c1`) + ); + +CREATE TABLE IF NOT EXISTS `testHash`( + `K` bigint, + `Q` varbinary(256), + `T` bigint, + `V` varbinary(1024), + INDEX i1(`K`, `V`) local, + PRIMARY KEY(`K`, `Q`, `T`) +) partition by hash(`K`) partitions 16; + +CREATE TABLE IF NOT EXISTS `testPartition` ( + `K` varbinary(1024), + `Q` varbinary(256), + `T` bigint, + `V` varbinary(1024), + K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4)), + PRIMARY KEY(`K`, `Q`, `T`) +) partition by key(K_PREFIX) partitions 15; + +CREATE TABLE IF NOT EXISTS `testPartitionHashComplex` ( + `c1` int NOT NULL, + `c2` bigint NOT NULL, + `c3` varchar(20) default NULL, +PRIMARY KEY (`c1`, `c2`) +) DEFAULT CHARSET = utf8mb4 COLLATE utf8mb4_bin ROW_FORMAT = DYNAMIC COMPRESSION = 'lz4_1.0' REPLICA_NUM = 3 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 10 +partition by hash(`c1`) subpartition by hash(`c2`) subpartitions 4 partitions 16; + +CREATE TABLE IF NOT EXISTS `testPartitionKeyComplex` ( + `c0` tinyint NOT NULL, + `c1` int NOT NULL, + `c2` bigint NOT NULL, + `c3` varbinary(1024) NOT NULL, + `c4` varchar(1024) NOT NULL, + `c5` varchar(1024) NOT NULL, + `c6` varchar(20) default NULL, +PRIMARY KEY (`c0`, `c1`, `c2`, `c3`, `c4`, `c5`) +) DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'lz4_1.0' REPLICA_NUM = 3 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 10 +partition by key(`c0`, `c1`, `c2`, `c3`, `c4`) subpartition by key(`c5`) subpartitions 4 partitions 16; + +CREATE TABLE IF NOT EXISTS `testDateTime` ( + `c0` DateTime NOT NULL, + `c1` Datetime(6) NOT NULL, + `c2` varchar(20) default NULL, +PRIMARY KEY (`c0`, `c1`) +) DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'lz4_1.0' REPLICA_NUM = 3 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 10 +PARTITION BY RANGE COLUMNS(`c0`) SUBPARTITION BY KEY(`c1`) ( + PARTITION `p0` VALUES LESS THAN ('2022-04-20 00:00:00') ( + SUBPARTITION `sp0`, SUBPARTITION `sp1`, SUBPARTITION `sp2`, SUBPARTITION `sp3` + ), + PARTITION `p1` VALUES LESS THAN ('2022-04-25 00:00:00') ( + SUBPARTITION `sp4`, SUBPARTITION `sp5`, SUBPARTITION `sp6`, SUBPARTITION `sp7` + ), + PARTITION `p2` VALUES LESS THAN ('2022-04-30 00:00:00') ( + SUBPARTITION `sp8`, SUBPARTITION `sp9`, SUBPARTITION `sp10`, SUBPARTITION `sp11` + ) +); + +CREATE TABLE IF NOT EXISTS `testPartitionRangeComplex` ( + `c1` int NOT NULL, + `c2` bigint NOT NULL, + `c3` varbinary(1024) NOT NULL, + `c4` varchar(1024) NOT NULL, + `c5` varchar(20) default NULL, +PRIMARY KEY (`c1`, `c2`, `c3`, `c4`) +) DEFAULT CHARSET = utf8mb4 ROW_FORMAT = DYNAMIC COMPRESSION = 'lz4_1.0' REPLICA_NUM = 3 BLOCK_SIZE = 16384 USE_BLOOM_FILTER = FALSE TABLET_SIZE = 134217728 PCTFREE = 10 +partition by range(`c1`) subpartition by range columns (`c2`, `c3`, `c4`) ( +PARTITION p0 VALUES LESS THAN (500) +( + SUBPARTITION p0sp0 VALUES LESS THAN (500, 't', 't'), + SUBPARTITION p0sp1 VALUES LESS THAN (1000, 'T', 'T'), + SUBPARTITION p0sp2 VALUES LESS THAN (MAXVALUE, MAXVALUE, MAXVALUE) +), +PARTITION p1 VALUES LESS THAN (1000) +( + SUBPARTITION p1sp0 VALUES LESS THAN (500, 't', 't'), + SUBPARTITION p1sp1 VALUES LESS THAN (1000, 'T', 'T'), + SUBPARTITION p1sp2 VALUES LESS THAN (MAXVALUE, MAXVALUE, MAXVALUE) +), +PARTITION p2 VALUES LESS THAN MAXVALUE +( + SUBPARTITION p2sp0 VALUES LESS THAN (500, 't', 't'), + SUBPARTITION p2sp1 VALUES LESS THAN (1000, 'T', 'T'), + SUBPARTITION p2sp2 VALUES LESS THAN (MAXVALUE, MAXVALUE, MAXVALUE) +)); + +CREATE TABLE IF NOT EXISTS `testKey` ( + `K` varbinary(1024), + `Q` varbinary(256), + `T` bigint, + `V` varbinary(1024), + INDEX i1(`K`, `V`) local, + PRIMARY KEY(`K`, `Q`, `T`) +) partition by key(K) partitions 15; + +CREATE TABLE IF NOT EXISTS `test_increment` ( + `c1` varchar(255), + `c2` int, + `c3` int, + PRIMARY KEY(`c1`) +); + +CREATE TABLE IF NOT EXISTS `test_append`( + `c1` varchar(255), + `c2` varbinary(1024), + `c3` varchar(255), + PRIMARY KEY(`c1`) +); + +CREATE TABLE IF NOT EXISTS `testRange` ( + `K` varbinary(1024), + `Q` varbinary(256), + `T` bigint, + `V` varbinary(10240), + INDEX i1(`K`, `V`) local, + PRIMARY KEY(`K`, `Q`, `T`) +) partition by range columns (`K`) ( + PARTITION p0 VALUES LESS THAN ('a'), + PARTITION p1 VALUES LESS THAN ('w'), + PARTITION p2 VALUES LESS THAN MAXVALUE +); + +CREATE TABLE IF NOT EXISTS `test_hbase$fn` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + KEY `idx_k_v` (`K`, `V`) LOCAL, + PRIMARY KEY (`K`, `Q`, `T`) +); + +CREATE TABLE `test_batch_query` ( + `c1` bigint NOT NULL, + `c2` varchar(20) DEFAULT NULL, +PRIMARY KEY (`c1`))partition by range(`c1`)(partition p0 values less than(200), partition p1 values less than(500), partition p2 values less than(900)); + +CREATE TABLE `test_query_filter_mutate` ( + `c1` bigint NOT NULL, + `c2` varbinary(1024) DEFAULT NULL, + `c3` varchar(20) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`c1`)) partition by range columns (`c1`) ( + PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), + PARTITION p2 VALUES LESS THAN MAXVALUE); + +CREATE TABLE `test_mutation_with_range` ( + `c1` bigint NOT NULL, + `c1sk` varchar(20) NOT NULL, + `c2` varbinary(1024) DEFAULT NULL, + `c3` varchar(20) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`c1`, `c1sk`)) partition by range columns (`c1`) ( + PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), + PARTITION p2 VALUES LESS THAN MAXVALUE); + +CREATE TABLE `test_mutation` ( + `c1` bigint NOT NULL, + `c2` varchar(20) NOT NULL, + `c3` varbinary(1024) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`c1`, `c2`)) partition by range columns (`c1`) ( + PARTITION p0 VALUES LESS THAN (300), + PARTITION p1 VALUES LESS THAN (1000), + PARTITION p2 VALUES LESS THAN MAXVALUE); + +CREATE TABLE `test_mutation_column_reverse` ( + `c2` bigint NOT NULL, + `c1` varchar(20) NOT NULL, + `c3` varbinary(1024) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`c2`, `c1`)) partition by range columns (`c2`) ( +PARTITION p0 VALUES LESS THAN (300), +PARTITION p1 VALUES LESS THAN (1000), +PARTITION p2 VALUES LESS THAN MAXVALUE); + +CREATE TABLE `test_throttle` ( + `c1` bigint NOT NULL, + `c2` varchar(20) NOT NULL, + `c3` varbinary(1024) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + PRIMARY KEY(`c1`, `c2`)) partition by range columns (`c1`) ( + PARTITION p0 VALUES LESS THAN (500000), + PARTITION p1 VALUES LESS THAN (1000000), + PARTITION p2 VALUES LESS THAN MAXVALUE); + +CREATE TABLE test_aggregation ( + `c1` varchar(255), + `c2` int NOT NULL, + `c3` bigint NOT NULL, + `c4` float NOT NULL, + `c5` double NOT NULL, + `c6` tinyint NULL, + `c7` datetime, + PRIMARY KEY(`c1`) +); + +CREATE TABLE `test_partition_aggregation` ( + `c1` bigint NOT NULL, + `c2` bigint DEFAULT NULL, + PRIMARY KEY (`c1`))partition by range(`c1`) ( + PARTITION p0 VALUES LESS THAN (200), + PARTITION p1 VALUES LESS THAN (500), + PARTITION p2 VALUES LESS THAN (900)); + +CREATE TABLE `test_ttl_timestamp` ( + `c1` bigint NOT NULL, + `c2` varchar(20) DEFAULT NULL, + `c3` bigint DEFAULT NULL, + `expired_ts` timestamp(6), +PRIMARY KEY (`c1`)) TTL(expired_ts + INTERVAL 0 SECOND); + +CREATE TABLE `test_ttl_timestamp_5s` ( + `c1` bigint NOT NULL, + `c2` varchar(20) DEFAULT NULL, + `c3` bigint DEFAULT NULL, + `expired_ts` timestamp(6), +PRIMARY KEY (`c1`)) TTL(expired_ts + INTERVAL 5 SECOND); + +CREATE TABLE IF NOT EXISTS `test_auto_increment_rowkey` ( + `c1` int auto_increment, + `c2` int NOT NULL, + `c3` int DEFAULT NULL, + `c4` varchar(255) DEFAULT NULL, + PRIMARY KEY(`c1`, `c2`)) partition by range columns(`c2`) ( + PARTITION p0 VALUES LESS THAN (100), + PARTITION p1 VALUES LESS THAN (1000)); + +CREATE TABLE IF NOT EXISTS `test_auto_increment_not_rowkey` ( + `c1` int NOT NULL, + `c2` int DEFAULT NULL, + `c3` tinyint auto_increment, + `c4` varchar(255) DEFAULT NULL, + PRIMARY KEY(`c1`)) partition by range columns(`c1`) ( + PARTITION p0 VALUES LESS THAN (100), + PARTITION p1 VALUES LESS THAN (1000)); + +CREATE TABLE IF NOT EXISTS `test_global_hash_range` ( + `C1` int(11) NOT NULL, + `C2` int(11) DEFAULT NULL, + `C3` int(11) DEFAULT NULL, + PRIMARY KEY (`C1`), + KEY `idx` (`C2`) GLOBAL partition by range(C2) ( + partition p0 values less than (100), + partition p1 values less than (200), + partition p2 values less than (300)), + KEY `idx2` (`C3`) LOCAL) partition by hash(c1) partitions 5; + +CREATE TABLE IF NOT EXISTS `test_global_hash_hash` ( + `c1` int(11) NOT NULL, + `c2` int(11) DEFAULT NULL, + `c3` varchar(255) DEFAULT NULL, + PRIMARY KEY (`c1`), + KEY `idx` (`c2`) GLOBAL partition by hash(`c2`) partitions 5) partition by hash(`c1`) partitions 7; + +CREATE TABLE IF NOT EXISTS `test_global_key_key` ( + `c1` int(11) NOT NULL, + `c2` int(11) DEFAULT NULL, + `c3` varchar(255) DEFAULT NULL, + PRIMARY KEY (`c1`), + KEY `idx` (`c2`) GLOBAL partition by key(`c2`) partitions 5) partition by key(`c1`) partitions 7; + +CREATE TABLE IF NOT EXISTS `test_global_range_range` ( + `c1` int(11) NOT NULL, + `c2` int(11) DEFAULT NULL, + `c3` varchar(255) DEFAULT NULL, + PRIMARY KEY (`c1`), + KEY `idx` (`c2`) GLOBAL partition by range(c2) ( + partition p0 values less than (100), + partition p1 values less than (200), + partition p2 values less than (1000))) partition by range(c1) ( + partition p0 values less than (100), + partition p1 values less than (200), + partition p2 values less than (1000)); + +CREATE TABLE IF NOT EXISTS `test_global_index_no_part` ( + `C1` int(11) NOT NULL, + `C2` int(11) DEFAULT NULL, + `C3` int(11) DEFAULT NULL, + PRIMARY KEY (`c1`), + KEY `idx` (`c2`) GLOBAL, + KEY `idx2` (c3) LOCAL) partition by hash(`c1`) partitions 7; + +CREATE TABLE IF NOT EXISTS `test_global_all_no_part` ( + `C1` int(11) NOT NULL, + `C2` int(11) DEFAULT NULL, + `C3` int(11) DEFAULT NULL, + PRIMARY KEY (`C1`), + KEY `idx` (`C2`) GLOBAL, + KEY `idx2` (C3) LOCAL); + +CREATE TABLE IF NOT EXISTS `test_global_primary_no_part` ( + `C1` int(11) NOT NULL, + `C2` int(11) DEFAULT NULL, + `C3` int(11) DEFAULT NULL, + PRIMARY KEY (`C1`), + KEY `idx` (`C2`) GLOBAL partition by hash(`C2`) partitions 5, + KEY `idx2` (C3) LOCAL); + +CREATE TABLE IF NOT EXISTS `test_ttl_timestamp_with_index` ( +`c1` varchar(20) NOT NULL, +`c2` bigint NOT NULL, +`c3` bigint DEFAULT NULL, +`c4` bigint DEFAULT NULL, +`expired_ts` timestamp(6), +PRIMARY KEY (`c1`, `c2`), +KEY `idx`(`c1`, `c4`) local, +KEY `idx2`(`c3`) global partition by hash(`c3`) partitions 4) +TTL(expired_ts + INTERVAL 0 SECOND) partition by key(`c1`) partitions 4; + +CREATE TABLE IF NOT EXISTS `error_message_table` ( + `c1` bigint(20) not null, + `c2` varchar(5) not null, + `c3` datetime default current_timestamp, + `c4` varchar(5) generated always as (SUBSTRING(c2, 1)), + `c5` double default 0, + PRIMARY KEY (`c1`)); + +CREATE TABLE IF NOT EXISTS `cse_index_1` ( + `measurement` VARBINARY(1024) NOT NULL, + `tag_key` VARBINARY(1024) NOT NULL, + `tag_value` VARBINARY(1024) NOT NULL, + `series_ids` MEDIUMBLOB NOT NULL, + PRIMARY KEY(`measurement`, `tag_key`, `tag_value`)) + partition by key(`measurement`) partitions 13; + +CREATE TABLE IF NOT EXISTS `test_auto_increment_one_rowkey` ( + `c1` int auto_increment, + `c2` int NOT NULL, PRIMARY KEY(`c1`)); + +CREATE TABLE IF NOT EXISTS `sync_item` ( + `uid` varchar(20) NOT NULL, + `object_id` varchar(32) NOT NULL, + `type` int(11) NULL, + `ver_oid` varchar(32) NULL, + `ver_ts` bigint(20) NULL, + `data_id` varchar(32) NULL, + CONSTRAINT `uid_object_id_unique` PRIMARY KEY (`uid`, `object_id`), + index idx1(`uid`, `type`) local) + DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci + PARTITION BY KEY(`uid`) PARTITIONS 32; + +CREATE TABLE `hash_key_sub_part` ( + `id` bigint(11) NOT NULL, + `uid` varchar(20) NOT NULL, + `object_id` varchar(32) NOT NULL, + `type` int(11) DEFAULT NULL, + `ver_oid` varchar(32) DEFAULT NULL, + `ver_ts` bigint(20) DEFAULT NULL, + `data_id` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`, `uid`, `object_id`) +) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_general_ci PARTITION BY HASH(`id`) SUBPARTITION BY KEY(`uid`) subpartitions 4 PARTITIONS 8; + +CREATE TABLE IF NOT EXISTS `test_table_object` ( + `c1` tinyint primary key, + `c2` smallint not null, + `c3` int not null, + `c4` bigint not null, + `c5` varchar(128) not null, + `c6` varbinary(128) not null, + `c7` float not null, + `c8` double not null, + `c9` timestamp(6) not null, + `c10` datetime(6) not null, + `c11` int default null, + `c12` tinytext DEFAULT NULL, + `c13` text DEFAULT NULL, + `c14` mediumtext DEFAULT NULL, + `c15` longtext DEFAULT NULL, + `c16` tinyblob DEFAULT NULL, + `c17` blob DEFAULT NULL, + `c18` mediumblob DEFAULT NULL, + `c19` longblob DEFAULT NULL +); + +CREATE TABLE `test_query_scan_order` ( + `c1` int(12) NOT NULL, + `c2` int(11) NOT NULL, + `c3` int(11) NOT NULL, + PRIMARY KEY (`c1`, `c2`), + KEY `idx` (`c1`, `c3`) BLOCK_SIZE 16384 LOCAL +) partition by hash(c1) +(partition `p0`, +partition `p1`, +partition `p2`); + +CREATE TABLE IF NOT EXISTS `test_put` ( + `id` varchar(20) NOT NULL, + `c1` bigint DEFAULT NULL, + `c2` bigint DEFAULT NULL, + `c3` varchar(32) DEFAULT NULL, + `c4` bigint DEFAULT NULL, + `expired_ts` timestamp(6) NOT NULL, + PRIMARY KEY(`id`)) TTL(expired_ts + INTERVAL 1 SECOND) PARTITION BY KEY(`id`) PARTITIONS 32; + +CREATE TABLE IF NOT EXISTS `test_put_with_local_index` ( + `id` varchar(20) NOT NULL, + `c1` bigint DEFAULT NULL, + index idx1(`c1`) local, + `expired_ts` timestamp(6) NOT NULL, + PRIMARY KEY(`id`)) TTL(expired_ts + INTERVAL 1 SECOND) PARTITION BY KEY(`id`) PARTITIONS 32; + +CREATE TABLE IF NOT EXISTS `test_put_with_global_index` ( + `id` varchar(20) NOT NULL, + `c1` bigint DEFAULT NULL, + index idx1(`c1`) global, + `expired_ts` timestamp(6) NOT NULL, + PRIMARY KEY(`id`)) TTL(expired_ts + INTERVAL 1 SECOND) PARTITION BY KEY(`id`) PARTITIONS 32; + +CREATE TABLEGROUP test SHARDING = 'ADAPTIVE'; +CREATE TABLE `test$family_group` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test; + +CREATE TABLEGROUP test2 SHARDING = 'ADAPTIVE'; +CREATE TABLE `test2$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test2; + +CREATE TABLEGROUP test_start_end_keys_key SHARDING = 'ADAPTIVE'; +CREATE TABLE `test_start_end_keys$family_key` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_start_end_keys_key +partition by key(K) partitions 17; + +CREATE TABLEGROUP test_start_end_keys_range SHARDING = 'ADAPTIVE'; +CREATE TABLE `test_start_end_keys_range$family_range` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_start_end_keys_range +partition by range columns (`K`) ( + PARTITION p0 VALUES LESS THAN ('a'), + PARTITION p1 VALUES LESS THAN ('w'), + PARTITION p2 VALUES LESS THAN MAXVALUE +); + +CREATE TABLE `test_batch_get` ( + `c1` int(11) NOT NULL, + `c2` int(11) NOT NULL, + `c3` int(11) DEFAULT NULL, + `c4` int(11) DEFAULT NULL, + PRIMARY KEY (`c1`, `c2`) +) partition by key(`c2`) partitions 3; + +CREATE TABLE `test_local_index_with_vgen_col` ( + `name` varchar(512) NOT NULL DEFAULT '', + `pk` varchar(512) NOT NULL, + `adiu` varchar(512) NOT NULL DEFAULT '', + `id` bigint(20) NOT NULL DEFAULT 0, + `name_v` varchar(20) GENERATED ALWAYS AS (substr(`name`,1,5)) VIRTUAL, + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`adiu`, `pk`, `gmt_create`), + KEY `idx_adiu_v_name` (`adiu`, `name_v`) BLOCK_SIZE 16384 LOCAL +) TTL (gmt_create + INTERVAL 300 SECOND) partition by key(adiu) partitions 8; + +CREATE TABLE `test_global_index_with_vgen_col` ( + `name` varchar(512) NOT NULL DEFAULT '', + `pk` varchar(512) NOT NULL, + `adiu` varchar(512) NOT NULL DEFAULT '', + `id` bigint(20) NOT NULL DEFAULT 0, + `name_v` varchar(20) GENERATED ALWAYS AS (substr(`name`,1,5)) VIRTUAL, + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`adiu`, `pk`, `gmt_create`), + KEY `idx_adiu_v_name` (`adiu`) global +) TTL (gmt_create + INTERVAL 300 SECOND) partition by key(adiu) partitions 8; + +CREATE TABLE `test_current_timestamp` ( + `c1` int not null, + `c2` varchar(255), + `c3` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`c1`), + KEY `idx_adiu_v_name` (`c2`, `c3`) global partition by key(`c2`) partitions 5 +) partition by key(`c1`) partitions 8; + +CREATE TABLE `table_ttl_00` ( + `name` varchar(512) NOT NULL, + `pk` varchar(512) NOT NULL, + `adiu` varchar(512) NOT NULL DEFAULT '', + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name_v` varchar(20) GENERATED ALWAYS AS (substr(`name`,1,5)) VIRTUAL, + `gmt_create` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`adiu`, `pk`), + KEY `idx_adiu` (`adiu`, `pk`, `name`) LOCAL, + KEY `idx_pk` (`pk`) GLOBAL +) TTL (gmt_create + INTERVAL 300 SECOND) partition by key(adiu) partitions 8; + +CREATE TABLE IF NOT EXISTS `audit_test` ( + `c1` bigint(20) not null, + `c2` varchar(50) default 'hello', + `c3` varchar(50) default 'world', + KEY `idx_c2` (`c2`) LOCAL, + KEY `idx_c3` (`c3`) LOCAL, + primary key (c1)); + + CREATE TABLE IF NOT EXISTS `test_p99` ( + `c1` bigint(20) NOT NULL, + `c2` bigint(20) DEFAULT NULL, + `c3` varchar(20) DEFAULT "hello", + PRIMARY KEY (`c1`) + ); + +alter system set kv_hotkey_throttle_threshold = 50; diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties deleted file mode 100644 index 1bdf1a15..00000000 --- a/src/test/resources/log4j.properties +++ /dev/null @@ -1,6 +0,0 @@ -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n \ No newline at end of file diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml new file mode 100644 index 00000000..cc211315 --- /dev/null +++ b/src/test/resources/logback-test.xml @@ -0,0 +1,12 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] %logger{50} - %msg%n + + + + + + + From 733bda4b2bae352d6828c345331df2565c35e01a Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 20 Aug 2024 16:32:43 +0800 Subject: [PATCH 2/5] correct reverse test --- .../java/com/alipay/oceanbase/rpc/ObTableCheckAndInsUpTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/rpc/ObTableCheckAndInsUpTest.java b/src/test/java/com/alipay/oceanbase/rpc/ObTableCheckAndInsUpTest.java index 9226b03e..dcc5ab21 100644 --- a/src/test/java/com/alipay/oceanbase/rpc/ObTableCheckAndInsUpTest.java +++ b/src/test/java/com/alipay/oceanbase/rpc/ObTableCheckAndInsUpTest.java @@ -259,7 +259,7 @@ public void testBatchWithReverseRowKwyColumn() throws Exception { Assert.assertTrue(false); } finally { Delete delete = client.delete(testTable); - delete.setRowKey(row(colVal("c1", 5L), colVal("c2", "c2_v0"))); + delete.setRowKey(row(colVal("c2", 5L), colVal("c1", "c2_v0"))); MutationResult res = delete.execute(); Assert.assertEquals(1, res.getAffectedRows()); } From cd4310a2c3d63d293516f82f3d525c4b158962bd Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 13 Sep 2024 15:30:24 +0800 Subject: [PATCH 3/5] format code --- .../alipay/oceanbase/rpc/ObTableClient.java | 96 ++++++++++--------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java index 95bf9fc4..f29020e9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java +++ b/src/main/java/com/alipay/oceanbase/rpc/ObTableClient.java @@ -538,8 +538,8 @@ private T execute(String tableName, TableExecuteCallback callback, ObServ if (odpMode) { obPair = new ObPair(0L, new ObTableParam(odpTable)); } else { - obPair = getTable(tableName, callback.getRowKey(), - needRefreshTableEntry, tableEntryRefreshIntervalWait, route); + obPair = getTable(tableName, callback.getRowKey(), needRefreshTableEntry, + tableEntryRefreshIntervalWait, route); } T t = callback.execute(obPair); resetExecuteContinuousFailureCount(tableName); @@ -608,7 +608,7 @@ private T execute(String tableName, TableExecuteCallback callback, ObServ } private abstract class MutationExecuteCallback { - private final Row rowKey; + private final Row rowKey; private final List keyRanges; MutationExecuteCallback(Row rowKey, List keyRanges) { @@ -702,11 +702,11 @@ private T executeMutation(String tableName, MutationExecuteCallback callb if (null != callback.getRowKey()) { // using row key obPair = getTable(tableName, callback.getRowKey(), needRefreshTableEntry, - tableEntryRefreshIntervalWait, route); + tableEntryRefreshIntervalWait, route); } else if (null != callback.getKeyRanges()) { // using scan range obPair = getTable(tableName, new ObTableQuery(), callback.getKeyRanges(), - needRefreshTableEntry, tableEntryRefreshIntervalWait, route); + needRefreshTableEntry, tableEntryRefreshIntervalWait, route); } else { throw new ObTableException("rowkey and scan range are null in mutation"); } @@ -741,14 +741,14 @@ private T executeMutation(String tableName, MutationExecuteCallback callb } } else { logger.warn("exhaust retry when replica not readable: {}", - ex.getMessage()); + ex.getMessage()); RUNTIME.error("replica not readable", ex); throw ex; } } else if (ex instanceof ObTableException && ((ObTableException) ex).isNeedRefreshTableEntry()) { // if the problem is the lack of row key name, throw directly - if(tableRowKeyElement.get(tableName) == null) { + if (tableRowKeyElement.get(tableName) == null) { throw ex; } needRefreshTableEntry = true; @@ -802,7 +802,6 @@ public void calculateContinuousFailure(String tableName, String errorMsg) throws } } - /** * Reset execute continuous failure count. * @param tableName table name @@ -1256,11 +1255,12 @@ private TableEntry refreshTableEntry(TableEntry tableEntry, String tableName, bo if (rowKeyElement != null) { tableEntry.setRowKeyElement(rowKeyElement); } else { - RUNTIME.error("partition table must add row key element name for table: " + tableName + " with table entry key: " - + tableEntryKey); + RUNTIME + .error("partition table must add row key element name for table: " + + tableName + " with table entry key: " + tableEntryKey); throw new ObTableEntryRefreshException( - "partition table must add row key element name for table: " + tableName + " with table entry key: " - + tableEntryKey); + "partition table must add row key element name for table: " + + tableName + " with table entry key: " + tableEntryKey); } } tableEntry.prepare(); @@ -1350,7 +1350,7 @@ private String refreshTableNameByTableGroup(String physicalTableName, String tab private long getPartition(TableEntry tableEntry, Row row) { // non partition if (!tableEntry.isPartitionTable() - || tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ZERO) { + || tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ZERO) { return 0L; } if (tableEntry.getPartitionInfo().getLevel() == ObPartitionLevel.LEVEL_ONE) { @@ -1449,9 +1449,8 @@ private ReplicaLocation getPartitionLocation(TableEntry tableEntry, long partId, * @return ObPair of partId and table * @throws Exception exception */ - public ObPair getTable(String tableName, Object[] rowKey, - boolean refresh, boolean waitForRefresh) - throws Exception { + public ObPair getTable(String tableName, Object[] rowKey, boolean refresh, + boolean waitForRefresh) throws Exception { ObServerRoute route = getRoute(false); return getTable(tableName, rowKey, refresh, waitForRefresh, route); } @@ -1472,7 +1471,7 @@ public ObPair getTable(String tableName, Object[] rowKey, bo TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh, false); Row row = new Row(); if (tableEntry.isPartitionTable() - && tableEntry.getPartitionInfo().getLevel() != ObPartitionLevel.LEVEL_ZERO) { + && tableEntry.getPartitionInfo().getLevel() != ObPartitionLevel.LEVEL_ZERO) { List curTableRowKeyNames = new ArrayList(); Map tableRowKeyEle = getRowKeyElement(tableName); if (tableRowKeyEle != null) { @@ -1486,8 +1485,7 @@ public ObPair getTable(String tableName, Object[] rowKey, bo for (int i = 0; i < rowKey.length; ++i) { if (i < curTableRowKeyNames.size()) { row.add(curTableRowKeyNames.get(i), rowKey[i]); - } - else { // the rowKey element in the table only contain partition key(s) or the input row key has redundant elements + } else { // the rowKey element in the table only contain partition key(s) or the input row key has redundant elements break; } } @@ -1575,7 +1573,7 @@ public ObPair getTable(String tableName, Row rowKey, boolean */ public ObPair getTable(String tableName, Row rowKey, boolean refresh, boolean waitForRefresh, ObServerRoute route) - throws Exception { + throws Exception { TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh, false); long partId; partId = getPartition(tableEntry, rowKey); // partition id in 3.x, origin partId in 4.x, logicId @@ -1594,9 +1592,11 @@ public ObPair getTable(String tableName, Row rowKey, boolean * @throws Exception exception */ public ObPair getTableWithPartId(String tableName, long partId, - boolean refresh, boolean waitForRefresh, boolean needFetchAll, - ObServerRoute route) throws Exception { - TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh, needFetchAll); + boolean refresh, boolean waitForRefresh, + boolean needFetchAll, ObServerRoute route) + throws Exception { + TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh, + needFetchAll); return getTableInternal(tableName, tableEntry, partId, waitForRefresh, route); } @@ -1738,11 +1738,11 @@ public List> getTables(String tableName, ObTableQuery * @return list of ObPair of partId(logicId) and tableParam * @throws Exception exception */ - public List> getTables(String tableName, ObTableQuery query, Object[] start, - boolean startInclusive, Object[] end, - boolean endInclusive, boolean refresh, - boolean waitForRefresh, ObServerRoute route) - throws Exception { + public List> getTables(String tableName, ObTableQuery query, + Object[] start, boolean startInclusive, + Object[] end, boolean endInclusive, + boolean refresh, boolean waitForRefresh, + ObServerRoute route) throws Exception { // 1. get TableEntry information TableEntry tableEntry = getOrRefreshTableEntry(tableName, refresh, waitForRefresh, false); @@ -1763,11 +1763,12 @@ public List> getTables(String tableName, ObTableQuery Row startRow = new Row(); Row endRow = new Row(); // ensure the format of column names and values if the current table is a table with partition - if (tableEntry.isPartitionTable() && tableEntry.getPartitionInfo().getLevel() != ObPartitionLevel.LEVEL_ZERO) { + if (tableEntry.isPartitionTable() + && tableEntry.getPartitionInfo().getLevel() != ObPartitionLevel.LEVEL_ZERO) { // scanRangeColumn may be longer than start/end in prefix scanning situation if (scanRangeColumns == null || scanRangeColumns.size() < start.length) { throw new IllegalArgumentException( - "length of key and scan range columns do not match, please use addRowKeyElement or set scan range columns"); + "length of key and scan range columns do not match, please use addRowKeyElement or set scan range columns"); } for (int i = 0; i < start.length; i++) { startRow.add(scanRangeColumns.get(i), start[i]); @@ -2033,8 +2034,8 @@ public ObPayload execute(ObPair obPair) throws Exception { request.setPartitionId(tableParam.getPartitionId()); ObPayload result = obTable.execute(request); String endpoint = obTable.getIp() + ":" + obTable.getPort(); - MonitorUtil.info(request, database, tableName, "UPDATE", endpoint, rowKey.getValues(), - (ObTableOperationResult) result, TableTime - start, + MonitorUtil.info(request, database, tableName, "UPDATE", endpoint, + rowKey.getValues(), (ObTableOperationResult) result, TableTime - start, System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); checkResult(obTable.getIp(), obTable.getPort(), request, result); return result; @@ -2107,14 +2108,15 @@ public ObPayload execute(ObPair obPair) throws Exception { ObTableParam tableParam = obPair.getRight(); ObTable obTable = tableParam.getObTable(); ObTableOperationRequest request = ObTableOperationRequest.getInstance( - tableName, DEL, rowKey.getValues(), null, null, obTable.getObTableOperationTimeout()); + tableName, DEL, rowKey.getValues(), null, null, + obTable.getObTableOperationTimeout()); request.setTableId(tableParam.getTableId()); // partId/tabletId request.setPartitionId(tableParam.getPartitionId()); ObPayload result = obTable.execute(request); String endpoint = obTable.getIp() + ":" + obTable.getPort(); - MonitorUtil.info(request, database, tableName, "DELETE", endpoint, rowKey.getValues(), - (ObTableOperationResult) result, TableTime - start, + MonitorUtil.info(request, database, tableName, "DELETE", endpoint, + rowKey.getValues(), (ObTableOperationResult) result, TableTime - start, System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); checkResult(obTable.getIp(), obTable.getPort(), request, result); return result; @@ -2196,8 +2198,8 @@ public ObPayload execute(ObPair obPair) throws Exception { request.setPartitionId(tableParam.getPartitionId()); ObPayload result = obTable.execute(request); String endpoint = obTable.getIp() + ":" + obTable.getPort(); - MonitorUtil.info(request, database, tableName, "INSERT", endpoint, rowKey.getValues(), - (ObTableOperationResult) result, TableTime - start, + MonitorUtil.info(request, database, tableName, "INSERT", endpoint, + rowKey.getValues(), (ObTableOperationResult) result, TableTime - start, System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); checkResult(obTable.getIp(), obTable.getPort(), request, result); return result; @@ -2237,8 +2239,8 @@ public ObPayload execute(ObPair obPair) throws Exception { request.setPartitionId(tableParam.getPartitionId()); ObPayload result = obTable.execute(request); String endpoint = obTable.getIp() + ":" + obTable.getPort(); - MonitorUtil.info(request, database, tableName, "PUT", endpoint, rowKey.getValues(), - (ObTableOperationResult) result, TableTime - start, + MonitorUtil.info(request, database, tableName, "PUT", endpoint, + rowKey.getValues(), (ObTableOperationResult) result, TableTime - start, System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); checkResult(obTable.getIp(), obTable.getPort(), request, result); return result; @@ -2320,8 +2322,8 @@ public ObPayload execute(ObPair obPair) throws Exception { request.setPartitionId(tableParam.getPartitionId()); ObPayload result = obTable.execute(request); String endpoint = obTable.getIp() + ":" + obTable.getPort(); - MonitorUtil.info(request, database, tableName, "REPLACE", endpoint, rowKey.getValues(), - (ObTableOperationResult) result, TableTime - start, + MonitorUtil.info(request, database, tableName, "REPLACE", endpoint, + rowKey.getValues(), (ObTableOperationResult) result, TableTime - start, System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); checkResult(obTable.getIp(), obTable.getPort(), request, result); return result; @@ -2515,8 +2517,8 @@ public ObPayload execute(ObPair obPair) throws Exception { request.setPartitionId(tableParam.getPartitionId()); ObPayload result = obTable.execute(request); String endpoint = obTable.getIp() + ":" + obTable.getPort(); - MonitorUtil.info(request, database, tableName, "INCREMENT", endpoint, rowKey.getValues(), - (ObTableOperationResult) result, TableTime - start, + MonitorUtil.info(request, database, tableName, "INCREMENT", endpoint, + rowKey.getValues(), (ObTableOperationResult) result, TableTime - start, System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); checkResult(obTable.getIp(), obTable.getPort(), request, result); return result; @@ -2594,8 +2596,8 @@ public ObPayload execute(ObPair obPair) throws Exception { request.setPartitionId(tableParam.getPartitionId()); ObPayload result = obTable.execute(request); String endpoint = obTable.getIp() + ":" + obTable.getPort(); - MonitorUtil.info(request, database, tableName, "INCREMENT", endpoint, rowKey.getValues(), - (ObTableOperationResult) result, TableTime - start, + MonitorUtil.info(request, database, tableName, "INCREMENT", endpoint, + rowKey.getValues(), (ObTableOperationResult) result, TableTime - start, System.currentTimeMillis() - TableTime, getslowQueryMonitorThreshold()); checkResult(obTable.getIp(), obTable.getPort(), request, result); return result; @@ -2867,8 +2869,8 @@ public ObPayload execute(final ObTableAbstractOperationRequest request) throws E } ObBorderFlag borderFlag = rang.getBorderFlag(); List> pairList = getTables(request.getTableName(), - tableQuery, start, borderFlag.isInclusiveStart(), end, - borderFlag.isInclusiveEnd(), false, false); + tableQuery, start, borderFlag.isInclusiveStart(), end, + borderFlag.isInclusiveEnd(), false, false); for (ObPair pair : pairList) { partIdMapObTable.put(pair.getLeft(), pair.getRight()); } From ff392dc1056c76d9c7911e9697c96832673b45a4 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 13 Sep 2024 15:56:35 +0800 Subject: [PATCH 4/5] check whether startRow and endRow contain the refColumn in Hash and Key part --- .../model/partition/ObHashPartDesc.java | 19 +++++++----- .../model/partition/ObKeyPartDesc.java | 31 ++++++++++++------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObHashPartDesc.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObHashPartDesc.java index e76011ef..969f81c0 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObHashPartDesc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObHashPartDesc.java @@ -130,15 +130,20 @@ public List getPartIds(Object startRowObj, boolean startInclusive, Object throw new IllegalArgumentException("rowkey length is " + startRow.size() + ", which is shortest than " + refIdx); } - // TODO: what if the curObRefColumnName does not exist in the startRow - if (startRow.get(curObRefColumnName) instanceof ObObj - && (((ObObj) startRow.get(curObRefColumnName)).isMinObj() || ((ObObj) startRow - .get(curObRefColumnName)).isMaxObj())) { + Object startValue = startRow.get(curObRefColumnName); + if (startValue == null) { + throw new IllegalArgumentException("Please include all partition key in start range. Currently missing key: { " + curObRefColumnName + " }"); + } + if (startValue instanceof ObObj + && (((ObObj) startValue).isMinObj() || ((ObObj) startValue).isMaxObj())) { return completeWorks; } - if (endRow.get(curObRefColumnName) instanceof ObObj - && (((ObObj) endRow.get(curObRefColumnName)).isMinObj() || ((ObObj) endRow - .get(curObRefColumnName)).isMaxObj())) { + Object endValue = endRow.get(curObRefColumnName); + if (endValue == null) { + throw new IllegalArgumentException("Please include all partition key in end range. Currently missing key: { " + curObRefColumnName + " }"); + } + if (endValue instanceof ObObj + && (((ObObj) endValue).isMinObj() || ((ObObj) endValue).isMaxObj())) { return completeWorks; } } diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java index eb153770..c4297168 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java @@ -103,8 +103,10 @@ public List getPartIds(Object startRowObj, boolean startInclusive, Object throw new IllegalArgumentException("length of start key and end key is not equal"); } - if (startRow.size() == 1 && startRow.getValues()[0] instanceof ObObj && ((ObObj) startRow.getValues()[0]).isMinObj() && - endRow.size() == 1 && endRow.getValues()[0] instanceof ObObj && ((ObObj) endRow.getValues()[0]).isMaxObj()) { + if (startRow.size() == 1 && startRow.getValues()[0] instanceof ObObj + && ((ObObj) startRow.getValues()[0]).isMinObj() && endRow.size() == 1 + && endRow.getValues()[0] instanceof ObObj + && ((ObObj) endRow.getValues()[0]).isMaxObj()) { return completeWorks; } @@ -116,14 +118,20 @@ public List getPartIds(Object startRowObj, boolean startInclusive, Object throw new IllegalArgumentException("rowkey length is " + startRow.size() + ", which is shortest than " + refIdx); } - if (startRow.get(curObRefColumnName) instanceof ObObj - && (((ObObj) startRow.get(curObRefColumnName)).isMinObj() || ((ObObj) startRow - .get(curObRefColumnName)).isMaxObj())) { + Object startValue = startRow.get(curObRefColumnName); + if (startValue == null) { + throw new IllegalArgumentException("Please include all partition key in start range. Currently missing key: { " + curObRefColumnName + " }"); + } + if (startValue instanceof ObObj + && (((ObObj) startValue).isMinObj() || ((ObObj) startValue).isMaxObj())) { return completeWorks; } - if (endRow.get(curObRefColumnName) instanceof ObObj - && (((ObObj) endRow.get(curObRefColumnName)).isMinObj() || ((ObObj) endRow - .get(curObRefColumnName)).isMaxObj())) { + Object endValue = endRow.get(curObRefColumnName); + if (endValue == null) { + throw new IllegalArgumentException("Please include all partition key in end range. Currently missing key: { " + curObRefColumnName + " }"); + } + if (endValue instanceof ObObj + && (((ObObj) endValue).isMinObj() || ((ObObj) endValue).isMaxObj())) { return completeWorks; } } @@ -229,14 +237,13 @@ private Long calcPartId(List evalValues) { long hashValue = 0L; for (int i = 0; i < partColumns.size(); i++) { - hashValue = ObHashUtils.toHashcode(evalValues.get(i), - partColumns.get(i), hashValue, - this.getPartFuncType()); + hashValue = ObHashUtils.toHashcode(evalValues.get(i), partColumns.get(i), hashValue, + this.getPartFuncType()); } hashValue = (hashValue > 0 ? hashValue : -hashValue); return ((long) partSpace << ObPartConstants.OB_PART_IDS_BITNUM) - | (hashValue % this.partNum); + | (hashValue % this.partNum); } private boolean equalsWithCollationType(ObCollationType collationType, Object s, Object t) From 1684393fbc7b69e7f10267711e4ed045fd57c7fa Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 13 Sep 2024 16:00:57 +0800 Subject: [PATCH 5/5] keep the same format with ObHashPartDesc --- .../rpc/location/model/partition/ObKeyPartDesc.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java index c4297168..b90553d9 100644 --- a/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java +++ b/src/main/java/com/alipay/oceanbase/rpc/location/model/partition/ObKeyPartDesc.java @@ -103,10 +103,8 @@ public List getPartIds(Object startRowObj, boolean startInclusive, Object throw new IllegalArgumentException("length of start key and end key is not equal"); } - if (startRow.size() == 1 && startRow.getValues()[0] instanceof ObObj - && ((ObObj) startRow.getValues()[0]).isMinObj() && endRow.size() == 1 - && endRow.getValues()[0] instanceof ObObj - && ((ObObj) endRow.getValues()[0]).isMaxObj()) { + if (startRow.size() == 1 && startRow.getValues()[0] instanceof ObObj && ((ObObj) startRow.getValues()[0]).isMinObj() && + endRow.size() == 1 && endRow.getValues()[0] instanceof ObObj && ((ObObj) endRow.getValues()[0]).isMaxObj()) { return completeWorks; }