diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 622a38c3159c16..28174ebf1eb9c9 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -388,6 +388,8 @@ Status retry_rpc(std::string_view op_name, const Request& req, Response* res, static_assert(std::is_base_of_v<::google::protobuf::Message, Request>); static_assert(std::is_base_of_v<::google::protobuf::Message, Response>); + const_cast(req).set_request_ip(BackendOptions::get_be_endpoint()); + int retry_times = 0; uint32_t duration_ms = 0; std::string error_msg; diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 7ecdf856659db0..4e90ec7f17d8cd 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -97,35 +97,44 @@ inline std::string encryt_sk(std::string debug_string) { template void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request* req) { if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side(); + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side(); + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() - << " table_id=" << req->table_id() << " tablet_id=" << req->tablet_id() - << " lock_id=" << req->lock_id() << " initiator=" << req->initiator() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " table_id=" << req->table_id() + << " tablet_id=" << req->tablet_id() << " lock_id=" << req->lock_id() + << " initiator=" << req->initiator() << " delete_bitmap_size=" << req->segment_delete_bitmaps_size(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " tablet_id=" << req->tablet_id() << " rowset_size=" << req->rowset_ids_size(); } else if constexpr (std::is_same_v) { - VLOG_DEBUG << "begin " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " tablet size: " << req->tablet_idx().size(); } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { - VLOG_DEBUG << "begin " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " request=" << req->ShortDebugString(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " tablet_id=" << req->tablet_id() << " rowset_size=" << req->rowset_ids_size(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() - << " table_id=" << req->table_id() << " lock_id=" << req->lock_id() - << " initiator=" << req->initiator() << " expiration=" << req->expiration() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " table_id=" << req->table_id() + << " lock_id=" << req->lock_id() << " initiator=" << req->initiator() + << " expiration=" << req->expiration() << " require_compaction_stats=" << req->require_compaction_stats(); } else { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " request=" << req->ShortDebugString(); } } @@ -138,21 +147,21 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re res->clear_partition_ids(); res->clear_versions(); } - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " response=" << res->ShortDebugString(); } else if constexpr (std::is_same_v) { if (res->status().code() != MetaServiceCode::OK) { res->clear_rowset_meta(); } - VLOG_DEBUG << "finish " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " status=" << res->status().ShortDebugString(); } else if constexpr (std::is_same_v) { - VLOG_DEBUG << "finish " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " status=" << res->status().ShortDebugString() << " tablet size: " << res->tablet_stats().size(); } else if constexpr (std::is_same_v || std::is_same_v) { - VLOG_DEBUG << "finish " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " response=" << res->ShortDebugString(); } else if constexpr (std::is_same_v) { if (res->status().code() != MetaServiceCode::OK) { @@ -161,7 +170,7 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re res->clear_versions(); res->clear_segment_delete_bitmaps(); } - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " status=" << res->status().ShortDebugString() << " tablet=" << res->tablet_id() << " delete_bitmap_count=" << res->segment_delete_bitmaps_size(); @@ -171,16 +180,16 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re res->clear_cumulative_compaction_cnts(); res->clear_cumulative_points(); } - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " status=" << res->status().ShortDebugString(); } else if constexpr (std::is_same_v || std::is_same_v) { std::string debug_string = encryt_sk(res->DebugString()); TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string); - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " response=" << debug_string; } else { - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " response=" << res->ShortDebugString(); } } diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 56060be67bae15..bc4a219d02691f 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -677,6 +677,7 @@ message ServiceRegistryPB { message BeginTxnRequest { optional string cloud_unique_id = 1; // For auth optional TxnInfoPB txn_info = 2; + optional string request_ip = 3; } message BeginTxnResponse { @@ -693,6 +694,7 @@ message PrecommitTxnRequest { optional int64 txn_id = 3; optional TxnCommitAttachmentPB commit_attachment = 4; optional int64 precommit_timeout_ms = 5; + optional string request_ip = 6; } message PrecommitTxnResponse { @@ -714,6 +716,7 @@ message CommitTxnRequest { optional bool is_txn_load = 9; repeated SubTxnInfo sub_txn_infos = 10; optional bool enable_txn_lazy_commit = 11; + optional string request_ip = 12; } message SubTxnInfo { @@ -747,6 +750,7 @@ message AbortTxnRequest { optional string label = 4; optional string reason = 5; optional TxnCommitAttachmentPB commit_attachment = 6; + optional string request_ip = 7; } message AbortTxnResponse { @@ -759,6 +763,7 @@ message GetTxnRequest { optional int64 db_id = 2; optional int64 txn_id = 3; optional string label = 4; + optional string request_ip = 5; } message GetTxnResponse { @@ -771,6 +776,7 @@ message GetTxnIdRequest { optional int64 db_id = 2; optional string label = 3; repeated TxnStatusPB txn_status = 4; + optional string request_ip = 5; } message GetTxnIdResponse { @@ -788,6 +794,7 @@ message BeginSubTxnRequest { repeated int64 table_ids = 5; // a random label used to generate a sub_txn_id optional string label = 6; + optional string request_ip = 7; } message BeginSubTxnResponse { @@ -806,6 +813,7 @@ message AbortSubTxnRequest { optional int64 db_id = 5; // set table_ids in txn_info repeated int64 table_ids = 6; + optional string request_ip = 7; } message AbortSubTxnResponse { @@ -815,6 +823,7 @@ message AbortSubTxnResponse { message GetCurrentMaxTxnRequest { optional string cloud_unique_id = 1; // For auth + optional string request_ip = 2; } message GetCurrentMaxTxnResponse { @@ -827,6 +836,7 @@ message AbortTxnWithCoordinatorRequest { optional string ip = 2; optional int64 id = 3; optional int64 start_time = 4; + optional string request_ip = 5; } message AbortTxnWithCoordinatorResponse { @@ -839,6 +849,7 @@ message CheckTxnConflictRequest { optional int64 end_txn_id = 3; repeated int64 table_ids = 4; optional bool ignore_timeout_txn = 5; + optional string request_ip = 6; } message CheckTxnConflictResponse { @@ -851,6 +862,7 @@ message CleanTxnLabelRequest { optional string cloud_unique_id = 1; // For auth optional int64 db_id = 2; repeated string labels = 3; + optional string request_ip = 4; } message CleanTxnLabelResponse { @@ -871,6 +883,8 @@ message GetVersionRequest { // True if get table version optional bool is_table_version = 9; + + optional string request_ip = 10; }; message GetVersionResponse { @@ -888,6 +902,7 @@ message GetVersionResponse { message GetObjStoreInfoRequest { optional string cloud_unique_id = 1; // For auth + optional string request_ip = 2; }; message AlterObjStoreInfoRequest { @@ -914,6 +929,7 @@ message AlterObjStoreInfoRequest { optional Operation op = 3; optional StorageVaultPB vault = 4; optional bool set_as_default_storage_vault = 5; + optional string request_ip = 6; } message AlterObjStoreInfoResponse { @@ -926,6 +942,7 @@ message UpdateAkSkRequest { optional string instance_id = 1; repeated RamUserPB internal_bucket_user = 2; optional RamUserPB ram_user = 3; + optional string request_ip = 4; } message UpdateAkSkResponse { @@ -946,6 +963,7 @@ message CreateTabletsRequest { repeated doris.TabletMetaCloudPB tablet_metas = 2; optional string storage_vault_name = 3; optional int64 db_id = 4; + optional string request_ip = 5; } message CreateTabletsResponse { @@ -957,6 +975,7 @@ message CreateTabletsResponse { message UpdateTabletRequest { optional string cloud_unique_id = 1; // For auth repeated TabletMetaInfoPB tablet_meta_infos = 2; + optional string request_ip = 3; } message UpdateTabletResponse { @@ -967,6 +986,7 @@ message UpdateTabletSchemaRequest { optional string cloud_unique_id = 1; // For auth optional int64 tablet_id = 2; optional doris.TabletSchemaCloudPB tablet_schema = 3; + optional string request_ip = 4; } message UpdateTabletSchemaResponse { @@ -976,12 +996,14 @@ message UpdateTabletSchemaResponse { message DropTabletRequest { optional string cloud_unique_id = 1; // For auth optional int64 tablet_id = 2; + optional string request_ip = 3; // TODO: There are more fields TBD } message GetTabletRequest { optional string cloud_unique_id = 1; // For auth optional int64 tablet_id = 2; + optional string request_ip = 3; // TODO: There are more fields TBD } @@ -996,6 +1018,7 @@ message CreateRowsetRequest { optional bool temporary = 3; optional int64 txn_id = 4; optional string tablet_job_id = 5; + optional string request_ip = 6; } message CreateRowsetResponse { @@ -1019,6 +1042,7 @@ message GetRowsetRequest { // returned schema format on rowset schema, used in variant type directly. // for compability reason we use FILL_WITH_DICT as default optional SchemaOp schema_op = 8 [default = FILL_WITH_DICT]; + optional string request_ip = 9; } message GetRowsetResponse { @@ -1032,6 +1056,7 @@ message GetRowsetResponse { message GetSchemaDictRequest { optional string cloud_unique_id = 1; // For auth optional int64 index_id = 2; + optional string request_ip = 3; } message GetSchemaDictResponse { @@ -1046,6 +1071,7 @@ message IndexRequest { optional int64 expiration = 4; optional int64 db_id = 5; optional bool is_new_table = 6; // For table version + optional string request_ip = 7; } message IndexResponse { @@ -1060,6 +1086,7 @@ message PartitionRequest { optional int64 expiration = 5; optional int64 db_id = 6; optional bool need_update_table_version = 7; + optional string request_ip = 8; } message PartitionResponse { @@ -1085,6 +1112,7 @@ message CreateInstanceRequest { optional RamUserPB ram_user = 5; optional bool sse_enabled = 6; optional StorageVaultPB vault = 7; + optional string request_ip = 8; } message CreateInstanceResponse { @@ -1105,6 +1133,7 @@ message AlterInstanceRequest { optional string instance_id = 1; optional Operation op = 2; optional string name = 3; + optional string request_ip = 4; } message AlterInstanceResponse { @@ -1114,6 +1143,7 @@ message AlterInstanceResponse { message GetInstanceRequest { optional string instance_id = 1; optional string cloud_unique_id = 2; + optional string request_ip = 3; } message GetInstanceResponse { @@ -1142,6 +1172,7 @@ message AlterClusterRequest { optional Operation op = 4; // for SQL mode rename cluster, rename to cluster name eq instance empty cluster name, need drop empty cluster optional bool replace_if_existing_empty_target_cluster = 5; + optional string request_ip = 6; } message AlterClusterResponse { @@ -1154,12 +1185,14 @@ message GetClusterRequest { optional string cluster_id = 3; optional string cluster_name = 4; optional string mysql_user_name = 5; + optional string request_ip = 6; } message GetClusterStatusRequest { repeated string instance_ids = 1; // Redundant field repeated string cloud_unique_ids = 2; optional ClusterStatus status = 3; + optional string request_ip = 4; } message GetClusterStatusResponse { @@ -1180,6 +1213,7 @@ message GetClusterResponse { message GetTabletStatsRequest { optional string cloud_unique_id = 1; repeated TabletIndexPB tablet_idx = 2; + optional string request_ip = 3; } message GetTabletStatsResponse { @@ -1190,6 +1224,7 @@ message GetTabletStatsResponse { message CreateStageRequest { optional string cloud_unique_id = 1; optional StagePB stage = 2; + optional string request_ip = 3; } message CreateStageResponse { @@ -1202,6 +1237,7 @@ message GetStageRequest { optional string mysql_user_name = 3; optional StagePB.StageType type = 4; optional string mysql_user_id = 5; + optional string request_ip = 6; } message GetStageResponse { @@ -1216,6 +1252,7 @@ message DropStageRequest { optional StagePB.StageType type = 4; optional string mysql_user_id = 5; optional string reason = 6; + optional string request_ip = 7; } message DropStageResponse { @@ -1224,6 +1261,7 @@ message DropStageResponse { message GetIamRequest { optional string cloud_unique_id = 1; + optional string request_ip = 2; } message GetIamResponse { @@ -1236,6 +1274,7 @@ message AlterIamRequest { optional string account_id = 1; optional string ak = 2; optional string sk = 3; + optional string request_ip = 4; } message AlterIamResponse { @@ -1245,6 +1284,7 @@ message AlterIamResponse { message AlterRamUserRequest { optional string instance_id = 1; optional RamUserPB ram_user = 2; + optional string request_ip = 3; } message AlterRamUserResponse { @@ -1254,6 +1294,7 @@ message AlterRamUserResponse { message StartTabletJobRequest { optional string cloud_unique_id = 1; // For auth optional TabletJobInfoPB job = 2; + optional string request_ip = 3; } message StartTabletJobResponse { @@ -1272,6 +1313,7 @@ message FinishTabletJobRequest { optional string cloud_unique_id = 1; // For auth optional Action action = 2; optional TabletJobInfoPB job = 3; + optional string request_ip = 4; } message FinishTabletJobResponse { @@ -1293,6 +1335,7 @@ message BeginCopyRequest { optional int64 file_num_limit = 10; optional int64 file_size_limit = 11; optional int64 file_meta_size_limit = 12; + optional string request_ip = 13; } message BeginCopyResponse { @@ -1315,6 +1358,7 @@ message FinishCopyRequest { optional int32 group_id = 6; optional Action action = 7; optional int64 finish_time_ms = 8; + optional string request_ip = 9; } message FinishCopyResponse { @@ -1327,6 +1371,7 @@ message GetCopyJobRequest { optional int64 table_id = 3; optional string copy_id = 4; optional int32 group_id = 5; + optional string request_ip = 6; } message GetCopyJobResponse { @@ -1338,6 +1383,7 @@ message GetCopyFilesRequest { optional string cloud_unique_id = 1; optional string stage_id = 2; optional int64 table_id = 3; + optional string request_ip = 4; } message GetCopyFilesResponse { @@ -1350,6 +1396,7 @@ message FilterCopyFilesRequest { optional string stage_id = 2; optional int64 table_id = 3; repeated ObjectFilePB object_files = 4; + optional string request_ip = 5; } message FilterCopyFilesResponse { @@ -1359,6 +1406,7 @@ message FilterCopyFilesResponse { message RecycleInstanceRequest { repeated string instance_ids = 1; + optional string request_ip = 2; } message StatisticsRecycleRequest { @@ -1466,6 +1514,7 @@ message UpdateDeleteBitmapRequest { // for load txn only optional int64 next_visible_version = 14; + optional string request_ip = 15; } message UpdateDeleteBitmapResponse { @@ -1482,6 +1531,7 @@ message GetDeleteBitmapRequest { optional int64 base_compaction_cnt = 7; optional int64 cumulative_compaction_cnt = 8; optional int64 cumulative_point = 9; + optional string request_ip = 10; } message GetDeleteBitmapResponse { @@ -1500,6 +1550,7 @@ message RemoveDeleteBitmapRequest { repeated string rowset_ids = 3; repeated int64 begin_versions = 4; repeated int64 end_versions = 5; + optional string request_ip = 6; } message RemoveDeleteBitmapResponse { @@ -1525,6 +1576,7 @@ message GetDeleteBitmapUpdateLockRequest { optional int64 expiration = 6; optional bool require_compaction_stats = 7 [default = false]; repeated TabletIndexPB tablet_indexes = 8; + optional string request_ip = 9; } message GetDeleteBitmapUpdateLockResponse { @@ -1541,6 +1593,7 @@ message RemoveDeleteBitmapUpdateLockRequest { optional int64 tablet_id = 3; optional int64 lock_id = 4; optional int64 initiator = 5; + optional string request_ip = 6; } message RemoveDeleteBitmapUpdateLockResponse { @@ -1551,6 +1604,7 @@ message GetRLTaskCommitAttachRequest { optional string cloud_unique_id = 1; // For auth optional int64 db_id = 2; optional int64 job_id = 3; + optional string request_ip = 4; } message GetRLTaskCommitAttachResponse { @@ -1563,6 +1617,7 @@ message ResetRLProgressRequest { optional int64 db_id = 2; optional int64 job_id = 3; map partition_to_offset = 4; + optional string request_ip = 5; } message ResetRLProgressResponse { @@ -1583,7 +1638,8 @@ message CheckKVRequest { } optional string cloud_unique_id = 1; // For auth optional CheckKeyInfos check_keys = 2; - optional Operation op = 3; + optional Operation op = 3; + optional string request_ip = 4; } message CheckKVResponse {