From 4a898e3bd417fcf3152d1f84f33e337778aee26f Mon Sep 17 00:00:00 2001 From: yiguolei Date: Fri, 21 Jun 2019 11:11:47 +0800 Subject: [PATCH 1/4] check file created and empty --- be/src/olap/rowset/segment_group.cpp | 8 ++++++++ be/src/olap/schema_change.cpp | 20 +++++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/be/src/olap/rowset/segment_group.cpp b/be/src/olap/rowset/segment_group.cpp index d9fa5acb42d90b..e6129fb9c8e7f2 100644 --- a/be/src/olap/rowset/segment_group.cpp +++ b/be/src/olap/rowset/segment_group.cpp @@ -373,6 +373,10 @@ bool SegmentGroup::index_loaded() { } OLAPStatus SegmentGroup::validate() { + if (!_file_created && !_empty || _file_created && _empty) { + LOG(FATAL) << "file created " << _file_created + << " empty " << _empty; + } if (_empty) { return OLAP_SUCCESS; } @@ -403,6 +407,10 @@ OLAPStatus SegmentGroup::validate() { } bool SegmentGroup::check() { + if (!_file_created && !_empty || _file_created && _empty) { + LOG(FATAL) << "file created " << _file_created + << " empty " << _empty; + } // if the segment group is converted from old files, _empty == false but _num_segments == 0 if (_empty && (_num_segments > 0 || !zero_num_rows())) { LOG(WARNING) << "invalid num segments for empty segment group, _num_segments:" << _num_segments diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 29c9c1890b1ae0..790c4c71194366 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1127,14 +1127,9 @@ OLAPStatus SchemaChangeHandler::process_alter_tablet(AlterTabletType type, << ", new_tablet_id=" << request.new_tablet_req.tablet_id << ", new_schema_hash=" << request.new_tablet_req.tablet_schema.schema_hash; OLAPStatus res = OLAP_SUCCESS; - // 1. Lock schema_change_lock util schema change info is stored in tablet header - if (!StorageEngine::instance()->tablet_manager()->try_schema_change_lock(request.base_tablet_id)) { - LOG(WARNING) << "failed to obtain schema change lock. " - << "base_tablet=" << request.base_tablet_id; - return OLAP_ERR_TRY_LOCK_FAILED; - } - // 2. Get base tablet + + // Get base tablet TabletSharedPtr base_tablet = StorageEngine::instance()->tablet_manager()->get_tablet( request.base_tablet_id, request.base_schema_hash); if (base_tablet == nullptr) { @@ -1143,6 +1138,17 @@ OLAPStatus SchemaChangeHandler::process_alter_tablet(AlterTabletType type, StorageEngine::instance()->tablet_manager()->release_schema_change_lock(request.base_tablet_id); return OLAP_ERR_TABLE_NOT_FOUND; } + ReadLock base_migration_rlock(base_tablet->get_migration_lock_ptr(), TRY_LOCK); + if (!base_migration_rlock.own_lock()) { + return OLAP_ERR_RWLOCK_ERROR; + } + + // Lock schema_change_lock util schema change info is stored in tablet header + if (!StorageEngine::instance()->tablet_manager()->try_schema_change_lock(request.base_tablet_id)) { + LOG(WARNING) << "failed to obtain schema change lock. " + << "base_tablet=" << request.base_tablet_id; + return OLAP_ERR_TRY_LOCK_FAILED; + } // 4. Returning success if new tablet already exist in StorageEngine. // It means that the current request was already handled. From 98bd7f3d069e24a100c4ac13f2222fc7828b1243 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Fri, 21 Jun 2019 12:06:45 +0800 Subject: [PATCH 2/4] Release schema change lock when get lock failed --- be/src/olap/schema_change.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 790c4c71194366..6a90d459cabbe4 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1128,6 +1128,12 @@ OLAPStatus SchemaChangeHandler::process_alter_tablet(AlterTabletType type, << ", new_schema_hash=" << request.new_tablet_req.tablet_schema.schema_hash; OLAPStatus res = OLAP_SUCCESS; + // Lock schema_change_lock util schema change info is stored in tablet header + if (!StorageEngine::instance()->tablet_manager()->try_schema_change_lock(request.base_tablet_id)) { + LOG(WARNING) << "failed to obtain schema change lock. " + << "base_tablet=" << request.base_tablet_id; + return OLAP_ERR_TRY_LOCK_FAILED; + } // Get base tablet TabletSharedPtr base_tablet = StorageEngine::instance()->tablet_manager()->get_tablet( @@ -1138,17 +1144,6 @@ OLAPStatus SchemaChangeHandler::process_alter_tablet(AlterTabletType type, StorageEngine::instance()->tablet_manager()->release_schema_change_lock(request.base_tablet_id); return OLAP_ERR_TABLE_NOT_FOUND; } - ReadLock base_migration_rlock(base_tablet->get_migration_lock_ptr(), TRY_LOCK); - if (!base_migration_rlock.own_lock()) { - return OLAP_ERR_RWLOCK_ERROR; - } - - // Lock schema_change_lock util schema change info is stored in tablet header - if (!StorageEngine::instance()->tablet_manager()->try_schema_change_lock(request.base_tablet_id)) { - LOG(WARNING) << "failed to obtain schema change lock. " - << "base_tablet=" << request.base_tablet_id; - return OLAP_ERR_TRY_LOCK_FAILED; - } // 4. Returning success if new tablet already exist in StorageEngine. // It means that the current request was already handled. @@ -1202,10 +1197,12 @@ OLAPStatus SchemaChangeHandler::process_alter_tablet(AlterTabletType type, ReadLock base_migration_rlock(base_tablet->get_migration_lock_ptr(), TRY_LOCK); if (!base_migration_rlock.own_lock()) { + StorageEngine::instance()->tablet_manager()->release_schema_change_lock(request.base_tablet_id); return OLAP_ERR_RWLOCK_ERROR; } ReadLock new_migration_rlock(new_tablet->get_migration_lock_ptr(), TRY_LOCK); if (!new_migration_rlock.own_lock()) { + StorageEngine::instance()->tablet_manager()->release_schema_change_lock(request.base_tablet_id); return OLAP_ERR_RWLOCK_ERROR; } From ee756e5388486840f0a15bd63ad6c55a8eb0e2cc Mon Sep 17 00:00:00 2001 From: yiguolei Date: Mon, 24 Jun 2019 09:45:21 +0800 Subject: [PATCH 3/4] Remove file created check --- be/src/olap/rowset/segment_group.cpp | 4 ---- be/src/olap/tablet_meta_manager.cpp | 1 - 2 files changed, 5 deletions(-) diff --git a/be/src/olap/rowset/segment_group.cpp b/be/src/olap/rowset/segment_group.cpp index e6129fb9c8e7f2..8061e05379f844 100644 --- a/be/src/olap/rowset/segment_group.cpp +++ b/be/src/olap/rowset/segment_group.cpp @@ -373,10 +373,6 @@ bool SegmentGroup::index_loaded() { } OLAPStatus SegmentGroup::validate() { - if (!_file_created && !_empty || _file_created && _empty) { - LOG(FATAL) << "file created " << _file_created - << " empty " << _empty; - } if (_empty) { return OLAP_SUCCESS; } diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp index f9f569db6887ef..47d9773d3b47ab 100755 --- a/be/src/olap/tablet_meta_manager.cpp +++ b/be/src/olap/tablet_meta_manager.cpp @@ -98,7 +98,6 @@ OLAPStatus TabletMetaManager::save(DataDir* store, key_stream << header_prefix << tablet_id << "_" << schema_hash; std::string key = key_stream.str(); VLOG(3) << "save tablet meta to meta store: key = " << key; - std::cout << "save tablet meta to meta store: key = " << key << std::endl; OlapMeta* meta = store->get_meta(); TabletMetaPB de_tablet_meta_pb; From bd41d0bff3a87cb387a06817c7952d3f57b4bb4e Mon Sep 17 00:00:00 2001 From: yiguolei Date: Mon, 24 Jun 2019 09:48:57 +0800 Subject: [PATCH 4/4] Remove file created check --- be/src/olap/rowset/segment_group.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/be/src/olap/rowset/segment_group.cpp b/be/src/olap/rowset/segment_group.cpp index 8061e05379f844..d9fa5acb42d90b 100644 --- a/be/src/olap/rowset/segment_group.cpp +++ b/be/src/olap/rowset/segment_group.cpp @@ -403,10 +403,6 @@ OLAPStatus SegmentGroup::validate() { } bool SegmentGroup::check() { - if (!_file_created && !_empty || _file_created && _empty) { - LOG(FATAL) << "file created " << _file_created - << " empty " << _empty; - } // if the segment group is converted from old files, _empty == false but _num_segments == 0 if (_empty && (_num_segments > 0 || !zero_num_rows())) { LOG(WARNING) << "invalid num segments for empty segment group, _num_segments:" << _num_segments